Merge branch 'master' into ui-restyle

api/src/com/cloud/network/NetworkModel.java

@@ -269,4 +269,6 @@ public interface NetworkModel {
     boolean getExecuteInSeqNtwkElmtCmd();
 
     boolean isNetworkReadyForGc(long networkId);
+
+    boolean getNetworkEgressDefaultPolicy(Long networkId);
 }

api/src/com/cloud/network/OvsProvider.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package com.cloud.network;
 
 import org.apache.cloudstack.api.Identity;

api/src/com/cloud/network/as/AutoScaleCounter.java

@@ -27,6 +27,8 @@ public class AutoScaleCounter {
 
         public static final AutoScaleCounterType Snmp = new AutoScaleCounterType("snmp");
         public static final AutoScaleCounterType Netscaler = new AutoScaleCounterType("netscaler");
+        public static final AutoScaleCounterType Cpu = new AutoScaleCounterType("cpu");
+        public static final AutoScaleCounterType Memory = new AutoScaleCounterType("memory");
 
         public AutoScaleCounterType(String name) {
             _name = name;

api/src/com/cloud/network/as/AutoScalePolicy.java

@@ -17,6 +17,8 @@
 
 package com.cloud.network.as;
 
+import java.util.Date;
+
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.InternalIdentity;
 
@@ -30,6 +32,8 @@ public interface AutoScalePolicy extends ControlledEntity, InternalIdentity {
     public int getDuration();
 
     public int getQuietTime();
+    
+	public Date getLastQuiteTime();
 
     public String getAction();
 

api/src/com/cloud/network/as/AutoScaleVmGroup.java

@@ -17,6 +17,8 @@
 
 package com.cloud.network.as;
 
+import java.util.Date;
+
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.InternalIdentity;
 
@@ -45,6 +47,8 @@ public interface AutoScaleVmGroup extends ControlledEntity, InternalIdentity {
 
     int getInterval();
 
+	Date getLastInterval();
+
     String getState();
 
     String getUuid();

api/src/com/cloud/network/as/Counter.java

@@ -23,7 +23,10 @@ import org.apache.cloudstack.api.InternalIdentity;
 public interface Counter extends InternalIdentity, Identity {
 
     public static enum Source {
-        netscaler, snmp
+        netscaler,
+        snmp,
+        cpu,
+        memory
     }
 
     String getName();
@@ -31,4 +34,5 @@ public interface Counter extends InternalIdentity, Identity {
     String getValue();
 
     Source getSource();
+    
 }

api/src/com/cloud/region/ha/GlobalLoadBalancerRule.java

@@ -50,9 +50,11 @@ public interface GlobalLoadBalancerRule extends Identity, InternalIdentity, Cont
     }
 
     enum ServiceType {
-        tcp, udp;
+        tcp, udp, http;
         public static boolean isValidServiceType(String serviceType) {
-            if (tcp.name().equalsIgnoreCase(serviceType) || udp.name().equalsIgnoreCase(serviceType)) {
+            if (tcp.name().equalsIgnoreCase(serviceType) ||
+                    udp.name().equalsIgnoreCase(serviceType) ||
+                    http.name().equalsIgnoreCase(serviceType)) {
                 return true;
             }
             return false;

api/src/org/apache/cloudstack/api/ApiConstants.java

@@ -502,6 +502,7 @@ public class ApiConstants {
     public static final String BAREMETAL_DISCOVER_NAME = "baremetaldiscovername";
     public static final String UCS_DN = "ucsdn";
     public static final String GSLB_PROVIDER = "gslbprovider";
+    public static final String EXCLUSIVE_GSLB_PROVIDER = "isexclusivegslbprovider";
     public static final String GSLB_PROVIDER_PUBLIC_IP = "gslbproviderpublicip";
     public static final String GSLB_PROVIDER_PRIVATE_IP = "gslbproviderprivateip";
     public static final String VM_SNAPSHOT_DESCRIPTION = "description";

api/src/org/apache/cloudstack/api/command/admin/router/ConfigureOvsElementCmd.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package org.apache.cloudstack.api.command.admin.router;
 
 import java.util.List;

api/src/org/apache/cloudstack/api/command/admin/router/ListOvsElementsCmd.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package org.apache.cloudstack.api.command.admin.router;
 
 import java.util.ArrayList;

api/src/org/apache/cloudstack/api/command/user/region/ha/gslb/CreateGlobalLoadBalancerRuleCmd.java

@@ -87,7 +87,7 @@ public class CreateGlobalLoadBalancerRuleCmd extends BaseAsyncCreateCmd {
     @Parameter(name = ApiConstants.GSLB_SERVICE_DOMAIN_NAME, type = CommandType.STRING, required = true, description = "domain name for the GSLB service.")
     private String serviceDomainName;
 
-    @Parameter(name = ApiConstants.GSLB_SERVICE_TYPE, type = CommandType.STRING, required = true, description = "GSLB service type (tcp, udp)")
+    @Parameter(name = ApiConstants.GSLB_SERVICE_TYPE, type = CommandType.STRING, required = true, description = "GSLB service type (tcp, udp, http)")
     private String serviceType;
 
     /////////////////////////////////////////////////////

api/src/org/apache/cloudstack/api/response/OvsProviderResponse.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package org.apache.cloudstack.api.response;
 
 import org.apache.cloudstack.api.ApiConstants;

client/tomcatconf/commands.properties.in

@@ -549,10 +549,6 @@ listSrxFirewalls=1
 listSrxFirewallNetworks=1
 
 #### Palo Alto firewall commands
-addExternalFirewall=1
-deleteExternalFirewall=1
-listExternalFirewalls=1
-
 addPaloAltoFirewall=1
 deletePaloAltoFirewall=1
 configurePaloAltoFirewall=1

core/src/com/cloud/agent/api/PerformanceMonitorAnswer.java

@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+public class PerformanceMonitorAnswer extends Answer {
+	public PerformanceMonitorAnswer() {
+	}
+
+	public PerformanceMonitorAnswer(PerformanceMonitorCommand cmd,
+			boolean result, String details) {
+		super(cmd, result, details);
+	}
+}

core/src/com/cloud/agent/api/PerformanceMonitorCommand.java

@@ -0,0 +1,46 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PerformanceMonitorCommand extends Command {
+
+	Map<String, String> params = new HashMap<String, String>();
+
+	public PerformanceMonitorCommand() {
+	}
+
+	public PerformanceMonitorCommand(Map<String, String> params, int wait) {
+		setWait(wait);
+		this.params = params;
+	}
+
+	@Override
+	public boolean executeInSequence() {
+		return false;
+	}
+
+	public Map<String, String> getParams() {
+		return params;
+	}
+
+	public void setParams(Map<String, String> params) {
+		this.params = params;
+	}
+}

engine/components-api/src/com/cloud/capacity/CapacityManager.java

@@ -91,4 +91,15 @@ public interface CapacityManager {
      * @return true if the count of host's running VMs >= hypervisor limit
      */
     boolean checkIfHostHasCpuCapability(long hostId, Integer cpuNum, Integer cpuSpeed);
+
+    /**
+     * Check if cluster will cross threshold if the cpu/memory requested are accomodated
+     * @param clusterId the clusterId to check
+     * @param cpuRequested cpu requested
+     * @param ramRequested cpu requested
+     * @return true if the customer crosses threshold, false otherwise
+     */
+    boolean checkIfClusterCrossesThreshold(Long clusterId, Integer cpuRequested, long ramRequested);
+
+    float getClusterOverProvisioningFactor(Long clusterId, short capacityType);
 }

engine/components-api/src/com/cloud/network/rules/FirewallManager.java

@@ -86,5 +86,5 @@ public interface FirewallManager extends FirewallService {
      */
     void removeRule(FirewallRule rule);
 
-    boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy) throws ResourceUnavailableException;
+    boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy, boolean add) throws ResourceUnavailableException;
 }

engine/orchestration/src/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java

@@ -1111,22 +1111,20 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
         }
 
         List<FirewallRuleVO> firewallEgressRulesToApply = _firewallDao.listByNetworkPurposeTrafficType(networkId, Purpose.Firewall, FirewallRule.TrafficType.Egress);
-        if (firewallEgressRulesToApply.size() == 0) {
-            NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId());
-            //there are no egress rules then apply the default egress rule
-            DataCenter zone = _dcDao.findById(network.getDataCenterId());
-            if (offering.getEgressDefaultPolicy() &&
+        NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId());
+        //there are no egress rules then apply the default egress rule
+        DataCenter zone = _dcDao.findById(network.getDataCenterId());
+        if ( _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) &&
                 _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) &&
                 (network.getGuestType() == Network.GuestType.Isolated || (network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced))) {
-                // add default egress rule to accept the traffic
-                _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), true);
-            }
-        } else {
-            if (!_firewallMgr.applyFirewallRules(firewallEgressRulesToApply, false, caller)) {
-                s_logger.warn("Failed to reapply firewall Egress rule(s) as a part of network id=" + networkId + " restart");
-                success = false;
-            }
+            // add default egress rule to accept the traffic
+            _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), offering.getEgressDefaultPolicy(), true);
         }
+        if (!_firewallMgr.applyFirewallRules(firewallEgressRulesToApply, false, caller)) {
+            s_logger.warn("Failed to reapply firewall Egress rule(s) as a part of network id=" + networkId + " restart");
+            success = false;
+        }
+
 
         // apply port forwarding rules
         if (!_rulesMgr.applyPortForwardingRulesForNetwork(networkId, false, caller)) {
@@ -2695,6 +2693,21 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
             s_logger.debug("Releasing " + firewallEgressRules.size() + " firewall egress rules for network id=" + networkId + " as a part of shutdownNetworkRules");
         }
 
+        try {
+            // delete default egress rule
+            DataCenter zone = _dcDao.findById(network.getDataCenterId());
+            if ( _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) &&
+                    (network.getGuestType() == Network.GuestType.Isolated || (network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced))) {
+                // add default egress rule to accept the traffic
+                _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), _networkModel.getNetworkEgressDefaultPolicy(networkId), false);
+            }
+
+        } catch (ResourceUnavailableException ex) {
+            s_logger.warn("Failed to cleanup firewall default egress rule as a part of shutdownNetworkRules due to ", ex);
+            success = false;
+        }
+
+
         for (FirewallRuleVO firewallRule : firewallEgressRules) {
             s_logger.trace("Marking firewall egress rule " + firewallRule + " with Revoke state");
             firewallRule.setState(FirewallRule.State.Revoke);

engine/schema/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -124,6 +124,7 @@
   <bean id="autoScalePolicyDaoImpl" class="com.cloud.network.as.dao.AutoScalePolicyDaoImpl" />
   <bean id="autoScaleVmGroupDaoImpl" class="com.cloud.network.as.dao.AutoScaleVmGroupDaoImpl" />
   <bean id="autoScaleVmGroupPolicyMapDaoImpl" class="com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDaoImpl" />
+  <bean id="autoScaleVmGroupVmMapDaoImpl" class="com.cloud.network.as.dao.AutoScaleVmGroupVmMapDaoImpl" />
   <bean id="autoScaleVmProfileDaoImpl" class="com.cloud.network.as.dao.AutoScaleVmProfileDaoImpl" />
   <bean id="capacityDaoImpl" class="com.cloud.capacity.dao.CapacityDaoImpl" />
   <bean id="certificateDaoImpl" class="com.cloud.certificate.dao.CertificateDaoImpl" />

engine/schema/src/com/cloud/capacity/dao/CapacityDao.java

@@ -45,11 +45,12 @@ public interface CapacityDao extends GenericDao<CapacityVO, Long> {
 
     Pair<List<Long>, Map<Long, Double>> orderPodsByAggregateCapacity(long zoneId, short capacityType);
 
-    List<SummedCapacity> findCapacityBy(Integer capacityType, Long zoneId, Long podId, Long clusterId, String resourceState);
+    List<SummedCapacity> findCapacityBy(Integer capacityType, Long zoneId,
+            Long podId, Long clusterId, String resourceState);
+    List<SummedCapacity> listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, Long clusterId, int level, Long limit);  
+    void updateCapacityState(Long dcId, Long podId, Long clusterId,
+            Long hostId, String capacityState);
+	List<Long> listClustersCrossingThreshold(short capacityType, Long zoneId, String ConfigName, long computeRequested);
 
-    List<SummedCapacity> listCapacitiesGroupedByLevelAndType(Integer capacityType, Long zoneId, Long podId, Long clusterId, int level, Long limit);
-
-    void updateCapacityState(Long dcId, Long podId, Long clusterId, Long hostId, String capacityState);
-
-    List<Long> listClustersCrossingThreshold(short capacityType, Long zoneId, String ConfigName, long computeRequested);
+    float findClusterConsumption(Long clusterId, short capacityType, long computeRequested);
 }

engine/schema/src/com/cloud/capacity/dao/CapacityDaoImpl.java

@@ -27,6 +27,9 @@ import java.util.Map;
 import javax.ejb.Local;
 import javax.inject.Inject;
 
+import com.cloud.dc.ClusterDetailsDao;
+import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
+import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.log4j.Logger;
 import org.springframework.stereotype.Component;
 
@@ -161,19 +164,24 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
     *     query from the configuration table
     *
     *     */
-    private static final String LIST_CLUSTERS_CROSSING_THRESHOLD =
-        "SELECT clusterList.cluster_id "
-            + "FROM (SELECT cluster.cluster_id cluster_id, ( (sum(cluster.used) + sum(cluster.reserved) + ?)/sum(cluster.total) ) ratio, cluster.configValue value "
-            + "FROM (SELECT capacity.cluster_id cluster_id, capacity.used_capacity used, capacity.reserved_capacity reserved, capacity.total_capacity * overcommit.value total, "
-            + "CASE (SELECT count(*) FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) "
-            + "WHEN 1 THEN (CASE WHEN (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ?) is NULL "
-            + "THEN (SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?)"
-            + "ELSE (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) END )"
-            + "ELSE (    SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?) " + "END configValue "
-            + "FROM `cloud`.`op_host_capacity` capacity INNER JOIN `cloud`.`cluster_details` overcommit ON overcommit.cluster_id = capacity.cluster_id "
-            + "WHERE capacity.data_center_id = ? AND capacity.capacity_type = ? AND capacity.total_capacity > 0 AND overcommit.name = ?) cluster " +
 
-            "GROUP BY cluster.cluster_id)  clusterList " + "WHERE clusterList.ratio > clusterList.value; ";
+    private static final String LIST_CLUSTERS_CROSSING_THRESHOLD = "SELECT clusterList.cluster_id " +
+                       "FROM (	SELECT cluster.cluster_id cluster_id, ( (sum(cluster.used) + sum(cluster.reserved) + ?)/sum(cluster.total) ) ratio, cluster.configValue value " +
+                                "FROM (	SELECT capacity.cluster_id cluster_id, capacity.used_capacity used, capacity.reserved_capacity reserved, capacity.total_capacity * overcommit.value total, " +
+                                            "CASE (SELECT count(*) FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) " +
+                                                "WHEN 1 THEN (	CASE WHEN (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ?) is NULL " +
+                                                                    "THEN (SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?)" +
+                                                                    "ELSE (SELECT details.value FROM `cloud`.`cluster_details` details WHERE details.cluster_id = capacity.cluster_id AND details.name = ? ) END )"  +
+                                                "ELSE (	SELECT config.value FROM `cloud`.`configuration` config WHERE config.name = ?) " +
+                                            "END configValue " +
+                                        "FROM `cloud`.`op_host_capacity` capacity INNER JOIN `cloud`.`cluster_details` overcommit ON overcommit.cluster_id = capacity.cluster_id " +
+                                        "WHERE capacity.data_center_id = ? AND capacity.capacity_type = ? AND capacity.total_capacity > 0 AND overcommit.name = ?) cluster " +
+
+                                "GROUP BY cluster.cluster_id)  clusterList " +
+                        "WHERE clusterList.ratio > clusterList.value; ";
+
+    private static final String  FIND_CLUSTER_CONSUMPTION_RATIO = "select ( (sum(capacity.used_capacity) + sum(capacity.reserved_capacity) + ?)/sum(capacity.total_capacity) ) " +
+            "from op_host_capacity capacity where cluster_id = ? and capacity_type = ?;";
 
     public CapacityDaoImpl() {
         _hostIdTypeSearch = createSearchBuilder();
@@ -883,4 +891,26 @@ public class CapacityDaoImpl extends GenericDaoBase<CapacityVO, Long> implements
             s_logger.warn("Error updating CapacityVO", e);
         }
     }
+
+    @Override
+    public float findClusterConsumption(Long clusterId, short capacityType, long computeRequested){
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        StringBuilder sql = new StringBuilder(FIND_CLUSTER_CONSUMPTION_RATIO);
+        PreparedStatement pstmt = null;
+        try {
+            pstmt = txn.prepareAutoCloseStatement(sql.toString());
+
+            pstmt.setLong(1, computeRequested);
+            pstmt.setLong(2, clusterId);
+            pstmt.setShort(3, capacityType);
+            ResultSet rs = pstmt.executeQuery();
+            while (rs.next()) {
+                return rs.getFloat(1);
+            }
+        } catch (Exception e) {
+            s_logger.warn("Error checking cluster threshold", e);
+        }
+        return 0;
+    }
+
 }

engine/schema/src/com/cloud/network/as/AutoScalePolicyVO.java

@@ -27,6 +27,8 @@ import javax.persistence.Id;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
 import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
 
 import org.apache.cloudstack.api.InternalIdentity;
 
@@ -57,6 +59,10 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity {
     @Column(name = "quiet_time", updatable = true, nullable = false)
     private int quietTime;
 
+	@Column(name = "last_quiet_time", updatable = true)
+	@Temporal(value = TemporalType.TIMESTAMP)
+	private Date lastQuiteTime;
+
     @Column(name = "action", updatable = false, nullable = false)
     private String action;
 
@@ -69,12 +75,14 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity {
     public AutoScalePolicyVO() {
     }
 
-    public AutoScalePolicyVO(long domainId, long accountId, int duration, int quietTime, String action) {
+	public AutoScalePolicyVO(long domainId, long accountId, int duration,
+			int quietTime, Date lastQuiteTime, String action) {
         this.uuid = UUID.randomUUID().toString();
         this.domainId = domainId;
         this.accountId = accountId;
         this.duration = duration;
         this.quietTime = quietTime;
+		this.lastQuiteTime = lastQuiteTime;
         this.action = action;
     }
 
@@ -113,6 +121,11 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity {
         return quietTime;
     }
 
+	@Override
+	public Date getLastQuiteTime() {
+		return lastQuiteTime;
+	}
+
     @Override
     public String getAction() {
         return action;
@@ -133,4 +146,8 @@ public class AutoScalePolicyVO implements AutoScalePolicy, InternalIdentity {
     public void setQuietTime(Integer quietTime) {
         this.quietTime = quietTime;
     }
+
+	public void setLastQuiteTime(Date lastQuiteTime) {
+		this.lastQuiteTime = lastQuiteTime;
+	}
 }

engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVO.java

@@ -27,6 +27,8 @@ import javax.persistence.Id;
 import javax.persistence.Inheritance;
 import javax.persistence.InheritanceType;
 import javax.persistence.Table;
+import javax.persistence.Temporal;
+import javax.persistence.TemporalType;
 
 import org.apache.cloudstack.api.InternalIdentity;
 
@@ -69,6 +71,10 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity {
     @Column(name = "interval")
     private int interval;
 
+	@Column(name = "last_interval", updatable = true)
+	@Temporal(value = TemporalType.TIMESTAMP)
+	private Date lastInterval;
+
     @Column(name = "profile_id")
     private long profileId;
 
@@ -84,8 +90,10 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity {
     public AutoScaleVmGroupVO() {
     }
 
-    public AutoScaleVmGroupVO(long lbRuleId, long zoneId, long domainId, long accountId, int minMembers, int maxMembers, int memberPort, int interval, long profileId,
-            String state) {
+	public AutoScaleVmGroupVO(long lbRuleId, long zoneId, long domainId,
+			long accountId, int minMembers, int maxMembers, int memberPort,
+			int interval, Date lastInterval, long profileId, String state) {
+
         this.uuid = UUID.randomUUID().toString();
         this.loadBalancerId = lbRuleId;
         this.minMembers = minMembers;
@@ -97,6 +105,7 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity {
         this.zoneId = zoneId;
         this.state = state;
         this.interval = interval;
+		this.lastInterval = lastInterval;
     }
 
     @Override
@@ -148,6 +157,11 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity {
         return interval;
     }
 
+	@Override
+	public Date getLastInterval() {
+		return lastInterval;
+	}
+
     @Override
     public long getProfileId() {
         return profileId;
@@ -182,6 +196,10 @@ public class AutoScaleVmGroupVO implements AutoScaleVmGroup, InternalIdentity {
         this.interval = interval;
     }
 
+	public void setLastInterval(Date lastInterval) {
+		this.lastInterval = lastInterval;
+	}
+
     public void setLoadBalancerId(Long loadBalancerId) {
         this.loadBalancerId = loadBalancerId;
     }

engine/schema/src/com/cloud/network/as/AutoScaleVmGroupVmMapVO.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.as;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+import org.apache.cloudstack.api.InternalIdentity;
+
+@Entity
+@Table(name = ("autoscale_vmgroup_vm_map"))
+public class AutoScaleVmGroupVmMapVO implements InternalIdentity {
+	@Id
+	@GeneratedValue(strategy = GenerationType.IDENTITY)
+	@Column(name = "id")
+	private long id;
+
+	@Column(name = "vmgroup_id")
+	private long vmGroupId;
+
+	@Column(name = "instance_id")
+	private long instanceId;
+
+	public AutoScaleVmGroupVmMapVO() {
+	}
+
+	public AutoScaleVmGroupVmMapVO(long vmGroupId, long instanceId) {
+		this.vmGroupId = vmGroupId;
+		this.instanceId = instanceId;
+	}
+
+	public long getId() {
+		return id;
+	}
+
+	public long getVmGroupId() {
+		return vmGroupId;
+	}
+
+	public long getInstanceId() {
+		return instanceId;
+	}
+
+}

engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDao.java

@@ -26,5 +26,7 @@ public interface AutoScalePolicyConditionMapDao extends GenericDao<AutoScalePoli
 
     public boolean isConditionInUse(Long conditionId);
 
-    boolean removeByAutoScalePolicyId(long id);
+	boolean removeByAutoScalePolicyId(long id);
+
+	List<AutoScalePolicyConditionMapVO> findByPolicyId(long id);
 }

engine/schema/src/com/cloud/network/as/dao/AutoScalePolicyConditionMapDaoImpl.java

@@ -56,6 +56,14 @@ public class AutoScalePolicyConditionMapDaoImpl extends GenericDaoBase<AutoScale
     public boolean removeByAutoScalePolicyId(long policyId) {
         SearchCriteria<AutoScalePolicyConditionMapVO> sc = createSearchCriteria();
         sc.addAnd("policyId", SearchCriteria.Op.EQ, policyId);
-        return expunge(sc) > 0;
-    }
+		return expunge(sc) > 0;
+	}
+
+	@Override
+	public List<AutoScalePolicyConditionMapVO> findByPolicyId(long policyId) {
+		SearchCriteria<AutoScalePolicyConditionMapVO> sc = createSearchCriteria();
+		sc.addAnd("policyId", SearchCriteria.Op.EQ, policyId);
+
+		return listBy(sc);
+	}
 }

engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDao.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.as.dao;
+
+import java.util.List;
+
+import com.cloud.network.as.AutoScaleVmGroupVmMapVO;
+import com.cloud.utils.db.GenericDao;
+
+public interface AutoScaleVmGroupVmMapDao extends GenericDao<AutoScaleVmGroupVmMapVO, Long> {
+	public Integer countByGroup(long vmGroupId);
+
+	public List<AutoScaleVmGroupVmMapVO> listByGroup(long vmGroupId);
+
+	public int remove(long vmGroupId, long vmId);
+
+}

engine/schema/src/com/cloud/network/as/dao/AutoScaleVmGroupVmMapDaoImpl.java

@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.as.dao;
+
+import java.util.List;
+
+import javax.ejb.Local;
+
+import org.springframework.stereotype.Component;
+
+import com.cloud.network.as.AutoScaleVmGroupVmMapVO;
+import com.cloud.utils.db.GenericDaoBase;
+import com.cloud.utils.db.SearchCriteria;
+
+@Component
+@Local(value={AutoScaleVmGroupVmMapDao.class})
+public class AutoScaleVmGroupVmMapDaoImpl extends GenericDaoBase<AutoScaleVmGroupVmMapVO, Long> implements AutoScaleVmGroupVmMapDao {
+
+	@Override
+	public Integer countByGroup(long vmGroupId) {
+
+		SearchCriteria<AutoScaleVmGroupVmMapVO> sc = createSearchCriteria();
+		sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId);
+		return getCount(sc);
+	}
+
+	@Override
+	public List<AutoScaleVmGroupVmMapVO> listByGroup(long vmGroupId) {
+		SearchCriteria<AutoScaleVmGroupVmMapVO> sc = createSearchCriteria();
+		sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId);
+		return listBy(sc);
+	}
+
+	@Override
+	public int remove(long vmGroupId, long vmId) {
+		SearchCriteria<AutoScaleVmGroupVmMapVO> sc = createSearchCriteria();
+		sc.addAnd("vmGroupId", SearchCriteria.Op.EQ, vmGroupId);
+		sc.addAnd("instanceId", SearchCriteria.Op.EQ, vmId);
+		return remove(sc);
+	}
+
+}

engine/schema/src/com/cloud/network/dao/ExternalLoadBalancerDeviceDaoImpl.java

@@ -57,6 +57,7 @@ public class ExternalLoadBalancerDeviceDaoImpl extends GenericDaoBase<ExternalLo
         allocationStateSearch.and("physicalNetworkId", allocationStateSearch.entity().getPhysicalNetworkId(), Op.EQ);
         allocationStateSearch.and("providerName", allocationStateSearch.entity().getProviderName(), Op.EQ);
         allocationStateSearch.and("allocationState", allocationStateSearch.entity().getAllocationState(), Op.EQ);
+        allocationStateSearch.and("exclusiveGslbProvider", allocationStateSearch.entity().getExclusiveGslbProvider(), Op.EQ);
         allocationStateSearch.done();
 
         deviceStatusSearch = createSearchBuilder();
@@ -99,6 +100,7 @@ public class ExternalLoadBalancerDeviceDaoImpl extends GenericDaoBase<ExternalLo
         sc.setParameters("physicalNetworkId", physicalNetworkId);
         sc.setParameters("providerName", provider_name);
         sc.setParameters("allocationState", state);
+        sc.setParameters("exclusiveGslbProvider", false);
         return search(sc, null);
     }
 

engine/schema/src/com/cloud/network/dao/ExternalLoadBalancerDeviceVO.java

@@ -76,6 +76,9 @@ public class ExternalLoadBalancerDeviceVO implements InternalIdentity, Identity
     @Column(name = "is_gslb_provider")
     private boolean gslbProvider;
 
+    @Column(name = "is_exclusive_gslb_provider")
+    private boolean exclusiveGslbProvider;
+
     @Column(name = "gslb_site_publicip")
     private String gslbSitePublicIP;
 
@@ -209,6 +212,14 @@ public class ExternalLoadBalancerDeviceVO implements InternalIdentity, Identity
         this.gslbProvider = gslbProvider;
     }
 
+    public boolean getExclusiveGslbProvider() {
+        return exclusiveGslbProvider;
+    }
+
+    public void setExclusiveGslbProvider(boolean exclusiveGslbProvider) {
+        this.exclusiveGslbProvider = exclusiveGslbProvider;
+    }
+
     public void setGslbSitePublicIP(String gslbSitePublicIP) {
         this.gslbSitePublicIP = gslbSitePublicIP;
     }

engine/schema/src/com/cloud/network/dao/OvsProviderDao.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package com.cloud.network.dao;
 
 import java.util.List;

engine/schema/src/com/cloud/network/dao/OvsProviderDaoImpl.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package com.cloud.network.dao;
 
 import java.util.List;

engine/schema/src/com/cloud/network/element/OvsProviderVO.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package com.cloud.network.element;
 
 import java.util.Date;

plugins/hypervisors/xen/src/com/cloud/hypervisor/xen/resource/CitrixResourceBase.java

@@ -168,6 +168,9 @@ import com.cloud.agent.api.OvsSetTagAndFlowAnswer;
 import com.cloud.agent.api.OvsFetchInterfaceAnswer;
 import com.cloud.agent.api.OvsSetTagAndFlowCommand;
 import com.cloud.agent.api.OvsFetchInterfaceCommand;
+import com.cloud.agent.api.PerformanceMonitorAnswer;
+import com.cloud.agent.api.PerformanceMonitorCommand;
+
 import com.cloud.agent.api.PingCommand;
 import com.cloud.agent.api.PingRoutingCommand;
 import com.cloud.agent.api.PingRoutingWithNwGroupsCommand;
@@ -304,7 +307,6 @@ import com.cloud.vm.snapshot.VMSnapshot;
 
 import com.cloud.utils.ssh.SSHCmdHelper;
 
-
 /**
  * CitrixResourceBase encapsulates the calls to the XenServer Xapi process
  * to perform the required functionalities for CloudStack.
@@ -657,11 +659,88 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
             return execute((PvlanSetupCommand)cmd);
         } else if (clazz == SetMonitorServiceCommand.class) {
             return execute((SetMonitorServiceCommand)cmd);
+		} else if (clazz == PerformanceMonitorCommand.class) {
+			return execute((PerformanceMonitorCommand) cmd);
         } else {
             return Answer.createUnsupportedCommandAnswer(cmd);
         }
     }
 
+	private Answer execute(PerformanceMonitorCommand cmd) {
+		Connection conn = getConnection();
+		String perfMon = getPerfMon(conn, cmd.getParams(), cmd.getWait());
+		if (perfMon == null) {
+			return new PerformanceMonitorAnswer(cmd, false, perfMon);
+		} else
+			return new PerformanceMonitorAnswer(cmd, true, perfMon);
+	}
+
+	private String getPerfMon(Connection conn, Map<String, String> params,
+			int wait) {
+		String result = null;
+		try {
+			result = callHostPluginAsync(conn, "vmopspremium", "asmonitor", 60,
+					params);
+			if (result != null)
+				return result;
+		} catch (Exception e) {
+			s_logger.error("Can not get performance monitor for AS due to ", e);
+		}
+		return null;
+	}
+
+	protected String callHostPluginAsync(Connection conn, String plugin,
+			String cmd, int wait, Map<String, String> params) {
+		int timeout = wait * 1000;
+		Map<String, String> args = new HashMap<String, String>();
+		Task task = null;
+		try {
+			for (String key : params.keySet()) {
+				args.put(key, params.get(key));
+			}
+			if (s_logger.isTraceEnabled()) {
+				s_logger.trace("callHostPlugin executing for command " + cmd
+						+ " with " + getArgsString(args));
+			}
+			Host host = Host.getByUuid(conn, _host.uuid);
+			task = host.callPluginAsync(conn, plugin, cmd, args);
+			// poll every 1 seconds
+			waitForTask(conn, task, 1000, timeout);
+			checkForSuccess(conn, task);
+			String result = task.getResult(conn);
+			if (s_logger.isTraceEnabled()) {
+				s_logger.trace("callHostPlugin Result: " + result);
+			}
+			return result.replace("<value>", "").replace("</value>", "")
+					.replace("\n", "");
+		} catch (Types.HandleInvalid e) {
+			s_logger.warn("callHostPlugin failed for cmd: " + cmd
+					+ " with args " + getArgsString(args)
+					+ " due to HandleInvalid clazz:" + e.clazz + ", handle:"
+					+ e.handle);
+		} catch (XenAPIException e) {
+			s_logger.warn(
+					"callHostPlugin failed for cmd: " + cmd + " with args "
+							+ getArgsString(args) + " due to " + e.toString(),
+					e);
+		} catch (XmlRpcException e) {
+			s_logger.warn(
+					"callHostPlugin failed for cmd: " + cmd + " with args "
+							+ getArgsString(args) + " due to " + e.getMessage(),
+					e);
+		} finally {
+			if (task != null) {
+				try {
+					task.destroy(conn);
+				} catch (Exception e1) {
+					s_logger.warn("unable to destroy task(" + task.toString()
+							+ ") on host(" + _host.uuid + ") due to ", e1);
+				}
+			}
+		}
+		return null;
+	}
+
     protected void scaleVM(Connection conn, VM vm, VirtualMachineTO vmSpec, Host host) throws XenAPIException, XmlRpcException {
 
         Long staticMemoryMax = vm.getMemoryStaticMax(conn);
@@ -5300,7 +5379,7 @@ public abstract class CitrixResourceBase implements ServerResource, HypervisorRe
 
                 com.trilead.ssh2.Session session = sshConnection.openSession();
 
-                String cmd = "mkdir -p /opt/cloud/bin";
+                String cmd = "mkdir -p /opt/cloud/bin /var/log/cloud";
                 if (!SSHCmdHelper.sshExecuteCmd(sshConnection, cmd)) {
                     throw new CloudRuntimeException("Cannot create directory /opt/cloud/bin on XenServer hosts");
                 }

plugins/network-elements/f5/src/com/cloud/network/element/F5ExternalLoadBalancerElement.java

@@ -321,7 +321,8 @@ public class F5ExternalLoadBalancerElement extends ExternalLoadBalancerDeviceMan
         pNetwork = physicalNetworks.get(0);
 
         String deviceType = NetworkDevice.F5BigIpLoadBalancer.getName();
-        lbDeviceVO = addExternalLoadBalancer(pNetwork.getId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceType, new F5BigIpResource(), false, null, null);
+        lbDeviceVO = addExternalLoadBalancer(pNetwork.getId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(),
+                deviceType, new F5BigIpResource(), false, false, null, null);
 
         if (lbDeviceVO != null) {
             lbHost = _hostDao.findById(lbDeviceVO.getHostId());
@@ -374,7 +375,8 @@ public class F5ExternalLoadBalancerElement extends ExternalLoadBalancerDeviceMan
             throw new InvalidParameterValueException("Invalid F5 load balancer device type");
         }
 
-        return addExternalLoadBalancer(cmd.getPhysicalNetworkId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceName, new F5BigIpResource(), false, null,
+        return addExternalLoadBalancer(cmd.getPhysicalNetworkId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(),
+                deviceName, new F5BigIpResource(), false, false, null,
             null);
 
     }

plugins/network-elements/juniper-srx/src/com/cloud/network/element/JuniperSRXExternalFirewallElement.java

@@ -218,6 +218,14 @@ public class JuniperSRXExternalFirewallElement extends ExternalFirewallDeviceMan
             return false;
         }
 
+        if (rules != null && rules.size() == 1 ) {
+            // for SRX no need to add default egress rule to DENY traffic
+            if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System &&
+                    ! _networkManager.getNetworkEgressDefaultPolicy(config.getId()))
+                return true;
+        }
+
+
         return applyFirewallRules(config, rules);
     }
 

plugins/network-elements/juniper-srx/src/com/cloud/network/resource/JuniperSrxResource.java

@@ -835,6 +835,15 @@ public class JuniperSrxResource implements ServerResource {
                 FirewallRule.FirewallRuleType type = rules[0].getType();
                 //getting
                 String guestCidr = rules[0].getGuestCidr();
+                List<String> cidrs = new ArrayList<String>();
+                cidrs.add(guestCidr);
+
+                List<Object[]> applications = new ArrayList<Object[]>();
+                Object[] application = new Object[3];
+                application[0] = Protocol.all;
+                application[1] = NetUtils.PORT_RANGE_MIN;
+                application[2] = NetUtils.PORT_RANGE_MAX;
+                applications.add(application);
 
                 for (String guestVlan : guestVlans) {
                     List<FirewallRuleTO> activeRulesForGuestNw = activeRules.get(guestVlan);
@@ -844,25 +853,24 @@ public class JuniperSrxResource implements ServerResource {
                     if (activeRulesForGuestNw.size() > 0 && type == FirewallRule.FirewallRuleType.User) {
                         addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS, guestVlan, extractApplications(activeRulesForGuestNw),
                             extractCidrs(activeRulesForGuestNw), defaultEgressPolicy);
+
+                      /* Adding default policy rules are required because the order of rules is important.
+                       * Depending on the rules order the traffic accept/drop is performed
+                       */
+                        removeEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, cidrs, defaultEgressPolicy);
+                        addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, applications, cidrs, defaultEgressPolicy);
                     }
 
-                    List<Object[]> applications = new ArrayList<Object[]>();
-                    Object[] application = new Object[3];
-                    application[0] = Protocol.all;
-                    application[1] = NetUtils.PORT_RANGE_MIN;
-                    application[2] = NetUtils.PORT_RANGE_MAX;
-                    applications.add(application);
-
-                    List<String> cidrs = new ArrayList<String>();
-                    cidrs.add(guestCidr);
                     //remove required with out comparing default policy  because in upgrade network offering we may required to delete
                     // the previously added rule
                     removeEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, cidrs, false);
-                    if (defaultEgressPolicy == true) {
-                        //add default egress security policy
-                        addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, applications, cidrs, false);
+                    if (defaultEgressPolicy == true && type == FirewallRule.FirewallRuleType.System) {
+                        removeEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, cidrs, defaultEgressPolicy);
+                        if (activeRulesForGuestNw.size() > 0) {
+                            //add default egress security policy
+                            addEgressSecurityPolicyAndApplications(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT, guestVlan, applications, cidrs, defaultEgressPolicy);
+                        }
                     }
-
                 }
                 commitConfiguration();
             } else {
@@ -2811,13 +2819,23 @@ public class JuniperSrxResource implements ServerResource {
                     xml = replaceXmlValue(xml, "src-address", srcAddrs);
                     dstAddrs = "<destination-address>any</destination-address>";
                     xml = replaceXmlValue(xml, "dst-address", dstAddrs);
-                    if (defaultEgressAction == true) {
-                        //configure block rules and default allow the traffic
-                        action = "<deny></deny>";
+                    if (type.equals(SecurityPolicyType.SECURITYPOLICY_EGRESS_DEFAULT)) {
+                        if (defaultEgressAction == false) {
+                            //for default policy is false add default deny rules
+                            action = "<deny></deny>";
+                        } else {
+                            action = "<permit></permit>";
+                        }
                     } else {
-                        action = "<permit></permit>";
+                        if (defaultEgressAction == true) {
+                            //configure egress rules to deny the traffic when default egress is allow
+                            action = "<deny></deny>";
+                        } else {
+                            action = "<permit></permit>";
+                        }
+
+                        xml = replaceXmlValue(xml, "action", action);
                     }
-                    xml = replaceXmlValue(xml, "action", action);
                 } else {
                     xml = replaceXmlValue(xml, "from-zone", fromZone);
                     xml = replaceXmlValue(xml, "to-zone", toZone);

plugins/network-elements/netscaler/src/com/cloud/api/commands/AddNetscalerLoadBalancerCmd.java

@@ -77,7 +77,7 @@ public class AddNetscalerLoadBalancerCmd extends BaseAsyncCmd {
                type = CommandType.BOOLEAN,
                required = false,
                description = "true if NetScaler device being added is for providing GSLB service")
-    private boolean isGslbProvider;
+    private Boolean isGslbProvider;
 
     @Parameter(name = ApiConstants.GSLB_PROVIDER_PUBLIC_IP, type = CommandType.STRING, required = false, description = "public IP of the site")
     private String gslbSitePublicIp;
@@ -85,6 +85,12 @@ public class AddNetscalerLoadBalancerCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.GSLB_PROVIDER_PRIVATE_IP, type = CommandType.STRING, required = false, description = "public IP of the site")
     private String gslbSitePrivateIp;
 
+    @Parameter(name = ApiConstants.EXCLUSIVE_GSLB_PROVIDER,
+            type = CommandType.BOOLEAN,
+            required = false,
+            description = "true if NetScaler device being added is for providing GSLB service exclusively and can not be used for LB")
+    private Boolean isExclusiveGslbProvider;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -110,7 +116,19 @@ public class AddNetscalerLoadBalancerCmd extends BaseAsyncCmd {
     }
 
     public boolean isGslbProvider() {
-        return isGslbProvider;
+        if (isGslbProvider != null) {
+            return isGslbProvider;
+        } else {
+            return false;
+        }
+    }
+
+    public boolean isExclusiveGslbProvider() {
+        if (isExclusiveGslbProvider != null) {
+            return isExclusiveGslbProvider;
+        } else {
+            return false;
+        }
     }
 
     public String getSitePublicIp() {

plugins/network-elements/netscaler/src/com/cloud/api/response/NetscalerLoadBalancerResponse.java

@@ -75,6 +75,10 @@ public class NetscalerLoadBalancerResponse extends BaseResponse {
     @Param(description = "true if NetScaler device is provisioned to be a GSLB service provider")
     private Boolean isGslbProvider;
 
+    @SerializedName(ApiConstants.EXCLUSIVE_GSLB_PROVIDER)
+    @Param(description = "true if NetScaler device is provisioned exclusively to be a GSLB service provider")
+    private Boolean isExclusiveGslbProvider;
+
     @SerializedName(ApiConstants.GSLB_PROVIDER_PUBLIC_IP)
     @Param(description = "public IP of the NetScaler representing GSLB site")
     private String gslbSitePublicIp;
@@ -137,6 +141,10 @@ public class NetscalerLoadBalancerResponse extends BaseResponse {
         this.isGslbProvider = isGslbProvider;
     }
 
+    public void setExclusiveGslbProvider(boolean isExclusiveGslbProvider) {
+        this.isExclusiveGslbProvider = isExclusiveGslbProvider;
+    }
+
     public void setGslbSitePublicIp(String publicIP) {
         this.gslbSitePublicIp = publicIP;
     }

plugins/network-elements/netscaler/src/com/cloud/network/element/NetscalerElement.java

@@ -381,9 +381,14 @@ public class NetscalerElement extends ExternalLoadBalancerDeviceManagerImpl impl
 
         }
 
+        if (cmd.isExclusiveGslbProvider() && !cmd.isGslbProvider()) {
+            throw new InvalidParameterValueException("NetScaler can be provisioned to be exclusive GSLB service provider" +
+                    " only if its being configured as GSLB service provider also.");
+        }
+
         ExternalLoadBalancerDeviceVO lbDeviceVO =
             addExternalLoadBalancer(cmd.getPhysicalNetworkId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceName, new NetscalerResource(),
-                cmd.isGslbProvider(), cmd.getSitePublicIp(), cmd.getSitePrivateIp());
+                cmd.isGslbProvider(), cmd.isExclusiveGslbProvider(), cmd.getSitePublicIp(), cmd.getSitePrivateIp());
 
         return lbDeviceVO;
     }
@@ -605,6 +610,7 @@ public class NetscalerElement extends ExternalLoadBalancerDeviceManagerImpl impl
         response.setObjectName("netscalerloadbalancer");
 
         response.setGslbProvider(lbDeviceVO.getGslbProvider());
+        response.setExclusiveGslbProvider(lbDeviceVO.getExclusiveGslbProvider());
         response.setGslbSitePublicIp(lbDeviceVO.getGslbSitePublicIP());
         response.setGslbSitePrivateIp(lbDeviceVO.getGslbSitePrivateIP());
 

plugins/network-elements/ovs/src/com/cloud/network/element/OvsElementService.java

@@ -1,3 +1,19 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
 package com.cloud.network.element;
 
 import com.cloud.utils.component.PluggableService;

plugins/network-elements/palo-alto/src/com/cloud/api/commands/AddExternalFirewallCmd.java

@@ -1,115 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.api.commands;
-
-import javax.inject.Inject;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.ApiErrorCode;
-import org.apache.cloudstack.api.BaseCmd;
-import org.apache.cloudstack.api.Parameter;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.ExternalFirewallResponse;
-import org.apache.cloudstack.api.response.ZoneResponse;
-
-import com.cloud.exception.InvalidParameterValueException;
-import com.cloud.host.Host;
-import com.cloud.network.element.PaloAltoFirewallElementService;
-import com.cloud.user.Account;
-import com.cloud.utils.exception.CloudRuntimeException;
-
-@APICommand(name = "addExternalFirewall", description = "Adds an external firewall appliance", responseObject = ExternalFirewallResponse.class)
-public class AddExternalFirewallCmd extends BaseCmd {
-    public static final Logger s_logger = Logger.getLogger(AddExternalFirewallCmd.class.getName());
-    private static final String s_name = "addexternalfirewallresponse";
-
-    /////////////////////////////////////////////////////
-    //////////////// API parameters /////////////////////
-    /////////////////////////////////////////////////////
-
-    @Parameter(name = ApiConstants.ZONE_ID,
-               type = CommandType.UUID,
-               entityType = ZoneResponse.class,
-               required = true,
-               description = "Zone in which to add the external firewall appliance.")
-    private Long zoneId;
-
-    @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "URL of the external firewall appliance.")
-    private String url;
-
-    @Parameter(name = ApiConstants.USERNAME, type = CommandType.STRING, required = true, description = "Username of the external firewall appliance.")
-    private String username;
-
-    @Parameter(name = ApiConstants.PASSWORD, type = CommandType.STRING, required = true, description = "Password of the external firewall appliance.")
-    private String password;
-
-    ///////////////////////////////////////////////////
-    /////////////////// Accessors ///////////////////////
-    /////////////////////////////////////////////////////
-
-    public Long getZoneId() {
-        return zoneId;
-    }
-
-    public String getUrl() {
-        return url;
-    }
-
-    public String getUsername() {
-        return username;
-    }
-
-    public String getPassword() {
-        return password;
-    }
-
-    /////////////////////////////////////////////////////
-    /////////////// API Implementation///////////////////
-    /////////////////////////////////////////////////////
-
-    @Inject
-    PaloAltoFirewallElementService _paElementService;
-
-    @Override
-    public String getCommandName() {
-        return s_name;
-    }
-
-    @Override
-    public long getEntityOwnerId() {
-        return Account.ACCOUNT_ID_SYSTEM;
-    }
-
-    @SuppressWarnings("deprecation")
-    @Override
-    public void execute() {
-        try {
-            Host externalFirewall = _paElementService.addExternalFirewall(this);
-            ExternalFirewallResponse response = _paElementService.createExternalFirewallResponse(externalFirewall);
-            response.setObjectName("externalfirewall");
-            response.setResponseName(getCommandName());
-            this.setResponseObject(response);
-        } catch (InvalidParameterValueException ipve) {
-            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, ipve.getMessage());
-        } catch (CloudRuntimeException cre) {
-            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, cre.getMessage());
-        }
-    }
-}

plugins/network-elements/palo-alto/src/com/cloud/api/commands/DeleteExternalFirewallCmd.java

@@ -1,89 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.api.commands;
-
-import javax.inject.Inject;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.ApiErrorCode;
-import org.apache.cloudstack.api.BaseCmd;
-import org.apache.cloudstack.api.Parameter;
-import org.apache.cloudstack.api.ServerApiException;
-import org.apache.cloudstack.api.response.HostResponse;
-import org.apache.cloudstack.api.response.SuccessResponse;
-
-import com.cloud.exception.InvalidParameterValueException;
-import com.cloud.network.element.PaloAltoFirewallElementService;
-import com.cloud.user.Account;
-
-@APICommand(name = "deleteExternalFirewall", description = "Deletes an external firewall appliance.", responseObject = SuccessResponse.class)
-public class DeleteExternalFirewallCmd extends BaseCmd {
-    public static final Logger s_logger = Logger.getLogger(DeleteExternalFirewallCmd.class.getName());
-    private static final String s_name = "deleteexternalfirewallresponse";
-
-    /////////////////////////////////////////////////////
-    //////////////// API parameters /////////////////////
-    /////////////////////////////////////////////////////
-
-    @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = HostResponse.class, required = true, description = "Id of the external firewall appliance.")
-    private Long id;
-
-    ///////////////////////////////////////////////////
-    /////////////////// Accessors ///////////////////////
-    /////////////////////////////////////////////////////
-
-    public Long getId() {
-        return id;
-    }
-
-    /////////////////////////////////////////////////////
-    /////////////// API Implementation///////////////////
-    /////////////////////////////////////////////////////
-
-    @Inject
-    PaloAltoFirewallElementService _paElementService;
-
-    @Override
-    public String getCommandName() {
-        return s_name;
-    }
-
-    @Override
-    public long getEntityOwnerId() {
-        return Account.ACCOUNT_ID_SYSTEM;
-    }
-
-    @SuppressWarnings("deprecation")
-    @Override
-    public void execute() {
-        try {
-            boolean result = _paElementService.deleteExternalFirewall(this);
-            if (result) {
-                SuccessResponse response = new SuccessResponse(getCommandName());
-                response.setResponseName(getCommandName());
-                this.setResponseObject(response);
-            } else {
-                throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete external firewall.");
-            }
-        } catch (InvalidParameterValueException e) {
-            throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Failed to delete external firewall.");
-        }
-    }
-}

plugins/network-elements/palo-alto/src/com/cloud/api/commands/ListExternalFirewallsCmd.java

@@ -1,89 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-package com.cloud.api.commands;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.inject.Inject;
-
-import org.apache.log4j.Logger;
-
-import org.apache.cloudstack.api.APICommand;
-import org.apache.cloudstack.api.ApiConstants;
-import org.apache.cloudstack.api.BaseListCmd;
-import org.apache.cloudstack.api.Parameter;
-import org.apache.cloudstack.api.command.user.offering.ListServiceOfferingsCmd;
-import org.apache.cloudstack.api.response.ExternalFirewallResponse;
-import org.apache.cloudstack.api.response.ListResponse;
-import org.apache.cloudstack.api.response.ZoneResponse;
-
-import com.cloud.host.Host;
-import com.cloud.network.element.PaloAltoFirewallElementService;
-
-@APICommand(name = "listExternalFirewalls", description = "List external firewall appliances.", responseObject = ExternalFirewallResponse.class)
-public class ListExternalFirewallsCmd extends BaseListCmd {
-    public static final Logger s_logger = Logger.getLogger(ListServiceOfferingsCmd.class.getName());
-    private static final String s_name = "listexternalfirewallsresponse";
-
-    /////////////////////////////////////////////////////
-    //////////////// API parameters /////////////////////
-    /////////////////////////////////////////////////////
-
-    @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, required = true, description = "zone Id")
-    private long zoneId;
-
-    /////////////////////////////////////////////////////
-    /////////////////// Accessors ///////////////////////
-    /////////////////////////////////////////////////////
-
-    public long getZoneId() {
-        return zoneId;
-    }
-
-    /////////////////////////////////////////////////////
-    /////////////// API Implementation///////////////////
-    /////////////////////////////////////////////////////
-
-    @Inject
-    PaloAltoFirewallElementService _paElementService;
-
-    @Override
-    public String getCommandName() {
-        return s_name;
-    }
-
-    @SuppressWarnings("deprecation")
-    @Override
-    public void execute() {
-
-        List<? extends Host> externalFirewalls = _paElementService.listExternalFirewalls(this);
-
-        ListResponse<ExternalFirewallResponse> listResponse = new ListResponse<ExternalFirewallResponse>();
-        List<ExternalFirewallResponse> responses = new ArrayList<ExternalFirewallResponse>();
-        for (Host externalFirewall : externalFirewalls) {
-            ExternalFirewallResponse response = _paElementService.createExternalFirewallResponse(externalFirewall);
-            response.setObjectName("externalfirewall");
-            response.setResponseName(getCommandName());
-            responses.add(response);
-        }
-
-        listResponse.setResponses(responses);
-        listResponse.setResponseName(getCommandName());
-        this.setResponseObject(listResponse);
-    }
-}

plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoExternalFirewallElement.java

@@ -32,12 +32,9 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
 import org.apache.cloudstack.network.ExternalNetworkDeviceManager.NetworkDevice;
 
 import com.cloud.api.ApiDBUtils;
-import com.cloud.api.commands.AddExternalFirewallCmd;
 import com.cloud.api.commands.AddPaloAltoFirewallCmd;
 import com.cloud.api.commands.ConfigurePaloAltoFirewallCmd;
-import com.cloud.api.commands.DeleteExternalFirewallCmd;
 import com.cloud.api.commands.DeletePaloAltoFirewallCmd;
-import com.cloud.api.commands.ListExternalFirewallsCmd;
 import com.cloud.api.commands.ListPaloAltoFirewallNetworksCmd;
 import com.cloud.api.commands.ListPaloAltoFirewallsCmd;
 import com.cloud.api.response.PaloAltoFirewallResponse;
@@ -293,85 +290,12 @@ public class PaloAltoExternalFirewallElement extends ExternalFirewallDeviceManag
         return true;
     }
 
-    @Override
-    @Deprecated
-    // should use more generic addNetworkDevice command to add firewall
-        public
-        Host addExternalFirewall(AddExternalFirewallCmd cmd) {
-        Long zoneId = cmd.getZoneId();
-        DataCenterVO zone = null;
-        PhysicalNetworkVO pNetwork = null;
-        HostVO fwHost = null;
-
-        zone = _dcDao.findById(zoneId);
-        if (zone == null) {
-            throw new InvalidParameterValueException("Could not find zone with ID: " + zoneId);
-        }
-
-        List<PhysicalNetworkVO> physicalNetworks = _physicalNetworkDao.listByZone(zoneId);
-        if ((physicalNetworks == null) || (physicalNetworks.size() > 1)) {
-            throw new InvalidParameterValueException("There are no physical networks or multiple physical networks configured in zone with ID: " + zoneId +
-                " to add this device.");
-        }
-        pNetwork = physicalNetworks.get(0);
-
-        String deviceType = NetworkDevice.PaloAltoFirewall.getName();
-        ExternalFirewallDeviceVO fwDeviceVO =
-            addExternalFirewall(pNetwork.getId(), cmd.getUrl(), cmd.getUsername(), cmd.getPassword(), deviceType, new PaloAltoResource());
-        if (fwDeviceVO != null) {
-            fwHost = _hostDao.findById(fwDeviceVO.getHostId());
-        }
-
-        return fwHost;
-    }
-
-    @Override
-    public boolean deleteExternalFirewall(DeleteExternalFirewallCmd cmd) {
-        return deleteExternalFirewall(cmd.getId());
-    }
-
-    @Override
-    @Deprecated
-    // should use more generic listNetworkDevice command
-        public
-        List<Host> listExternalFirewalls(ListExternalFirewallsCmd cmd) {
-        List<Host> firewallHosts = new ArrayList<Host>();
-        Long zoneId = cmd.getZoneId();
-        DataCenterVO zone = null;
-        PhysicalNetworkVO pNetwork = null;
-
-        if (zoneId != null) {
-            zone = _dcDao.findById(zoneId);
-            if (zone == null) {
-                throw new InvalidParameterValueException("Could not find zone with ID: " + zoneId);
-            }
-
-            List<PhysicalNetworkVO> physicalNetworks = _physicalNetworkDao.listByZone(zoneId);
-            if ((physicalNetworks == null) || (physicalNetworks.size() > 1)) {
-                throw new InvalidParameterValueException("There are no physical networks or multiple physical networks configured in zone with ID: " + zoneId +
-                    " to add this device.");
-            }
-            pNetwork = physicalNetworks.get(0);
-        }
-
-        firewallHosts.addAll(listExternalFirewalls(pNetwork.getId(), NetworkDevice.PaloAltoFirewall.getName()));
-        return firewallHosts;
-    }
-
-    @Override
-    public ExternalFirewallResponse createExternalFirewallResponse(Host externalFirewall) {
-        return super.createExternalFirewallResponse(externalFirewall);
-    }
-
     @Override
     public List<Class<?>> getCommands() {
         List<Class<?>> cmdList = new ArrayList<Class<?>>();
-        cmdList.add(AddExternalFirewallCmd.class);
         cmdList.add(AddPaloAltoFirewallCmd.class);
         cmdList.add(ConfigurePaloAltoFirewallCmd.class);
-        cmdList.add(DeleteExternalFirewallCmd.class);
         cmdList.add(DeletePaloAltoFirewallCmd.class);
-        cmdList.add(ListExternalFirewallsCmd.class);
         cmdList.add(ListPaloAltoFirewallNetworksCmd.class);
         cmdList.add(ListPaloAltoFirewallsCmd.class);
         return cmdList;

plugins/network-elements/palo-alto/src/com/cloud/network/element/PaloAltoFirewallElementService.java

@@ -20,12 +20,9 @@ import java.util.List;
 
 import org.apache.cloudstack.api.response.ExternalFirewallResponse;
 
-import com.cloud.api.commands.AddExternalFirewallCmd;
 import com.cloud.api.commands.AddPaloAltoFirewallCmd;
 import com.cloud.api.commands.ConfigurePaloAltoFirewallCmd;
-import com.cloud.api.commands.DeleteExternalFirewallCmd;
 import com.cloud.api.commands.DeletePaloAltoFirewallCmd;
-import com.cloud.api.commands.ListExternalFirewallsCmd;
 import com.cloud.api.commands.ListPaloAltoFirewallNetworksCmd;
 import com.cloud.api.commands.ListPaloAltoFirewallsCmd;
 import com.cloud.api.response.PaloAltoFirewallResponse;
@@ -72,24 +69,4 @@ public interface PaloAltoFirewallElementService extends PluggableService {
     public List<? extends Network> listNetworks(ListPaloAltoFirewallNetworksCmd cmd);
 
     public PaloAltoFirewallResponse createPaloAltoFirewallResponse(ExternalFirewallDeviceVO fwDeviceVO);
-
-    @Deprecated
-    // API helper function supported for backward compatibility
-        public
-        Host addExternalFirewall(AddExternalFirewallCmd cmd);
-
-    @Deprecated
-    // API helper function supported for backward compatibility
-        public
-        boolean deleteExternalFirewall(DeleteExternalFirewallCmd cmd);
-
-    @Deprecated
-    // API helper function supported for backward compatibility
-        public
-        List<Host> listExternalFirewalls(ListExternalFirewallsCmd cmd);
-
-    @Deprecated
-    // API helper function supported for backward compatibility
-        public
-        ExternalFirewallResponse createExternalFirewallResponse(Host externalFirewall);
 }

plugins/network-elements/palo-alto/src/com/cloud/network/resource/PaloAltoResource.java

@@ -29,6 +29,7 @@ import java.util.Map;
 
 import javax.naming.ConfigurationException;
 import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.dom.DOMSource;
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Source;
 import javax.xml.transform.Transformer;
@@ -55,6 +56,7 @@ import org.apache.http.message.BasicNameValuePair;
 import org.apache.http.protocol.HTTP;
 import org.apache.log4j.Logger;
 import org.w3c.dom.Document;
+import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
 import org.xml.sax.InputSource;
 
@@ -462,11 +464,14 @@ public class PaloAltoResource implements ServerResource {
             String guestVlanSubnet = NetUtils.getCidrSubNet(guestVlanGateway, cidrSize);
 
             Long publicVlanTag = null;
-            if (ip.getBroadcastUri() != null && !ip.getBroadcastUri().equals("untagged")) {
-                try {
-                    publicVlanTag = Long.parseLong(ip.getBroadcastUri());
-                } catch (Exception e) {
-                    throw new ExecutionException("Could not parse public VLAN tag: " + ip.getBroadcastUri());
+            if (ip.getBroadcastUri() != null) {
+                String parsedVlanTag = parsePublicVlanTag(ip.getBroadcastUri());
+                if (!parsedVlanTag.equals("untagged")) {
+                    try {
+                        publicVlanTag = Long.parseLong(parsedVlanTag);
+                    } catch (Exception e) {
+                        throw new ExecutionException("Could not parse public VLAN tag: " + parsedVlanTag);
+                    }
                 }
             }
 
@@ -520,6 +525,9 @@ public class PaloAltoResource implements ServerResource {
         String privateGateway, String privateSubnet, long privateCidrSize) throws ExecutionException {
         privateSubnet = privateSubnet + "/" + privateCidrSize;
 
+        // remove any orphaned egress rules if they exist...
+        removeOrphanedFirewallRules(cmdList, privateVlanTag);
+
         if (type.equals(GuestNetworkType.SOURCE_NAT)) {
             manageNetworkIsolation(cmdList, PaloAltoPrimative.DELETE, privateVlanTag, privateSubnet, privateGateway);
             manageSrcNatRule(cmdList, PaloAltoPrimative.DELETE, type, publicVlanTag, sourceNatIpAddress + "/32", privateVlanTag, privateGateway + "/" + privateCidrSize);
@@ -943,11 +951,16 @@ public class PaloAltoResource implements ServerResource {
         String dstNatName = genDstNatRuleName(publicIp, rule.getId());
 
         String publicInterfaceName;
-        String publicVlanTag = rule.getSrcVlanTag();
-        if (publicVlanTag == null || publicVlanTag.equals("untagged")) {
+        String publicVlanTag;
+        if (rule.getSrcVlanTag() == null) {
             publicInterfaceName = genPublicInterfaceName(new Long("9999"));
         } else {
-            publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag));
+            publicVlanTag = parsePublicVlanTag(rule.getSrcVlanTag());
+            if (publicVlanTag.equals("untagged")) {
+                publicInterfaceName = genPublicInterfaceName(new Long("9999"));
+            } else {
+                publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag));
+            }
         }
 
         switch (prim) {
@@ -1085,11 +1098,16 @@ public class PaloAltoResource implements ServerResource {
         String stcNatName = genStcNatRuleName(publicIp, rule.getId());
 
         String publicInterfaceName;
-        String publicVlanTag = rule.getSrcVlanTag();
-        if (publicVlanTag == null || publicVlanTag.equals("untagged")) {
+        String publicVlanTag;
+        if (rule.getSrcVlanTag() == null) {
             publicInterfaceName = genPublicInterfaceName(new Long("9999"));
         } else {
-            publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag));
+            publicVlanTag = parsePublicVlanTag(rule.getSrcVlanTag());
+            if (publicVlanTag.equals("untagged")) {
+                publicInterfaceName = genPublicInterfaceName(new Long("9999"));
+            } else {
+                publicInterfaceName = genPublicInterfaceName(new Long(publicVlanTag));
+            }
         }
 
         switch (prim) {
@@ -1170,13 +1188,20 @@ public class PaloAltoResource implements ServerResource {
     /*
      * Firewall rule implementation
      */
-
-    private String genFirewallRuleName(long id) {
+    private String genFirewallRuleName(long id) { // ingress
         return "policy_" + Long.toString(id);
     }
+    private String genFirewallRuleName(long id, String vlan) { // egress
+        return "policy_"+Long.toString(id)+"_"+vlan;
+    }
 
     public boolean manageFirewallRule(ArrayList<IPaloAltoCommand> cmdList, PaloAltoPrimative prim, FirewallRuleTO rule) throws ExecutionException {
-        String ruleName = genFirewallRuleName(rule.getId());
+        String ruleName;
+        if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) {
+            ruleName = genFirewallRuleName(rule.getId(), rule.getSrcVlanTag());
+        } else {
+            ruleName = genFirewallRuleName(rule.getId());
+        }
 
         switch (prim) {
 
@@ -1203,6 +1228,7 @@ public class PaloAltoResource implements ServerResource {
                 String serviceXML;
 
                 String protocol = rule.getProtocol();
+                String action = "allow";
 
                 // Only ICMP will use an Application, so others will be any.
                 if (protocol.equals(Protocol.ICMP.toString())) {
@@ -1232,11 +1258,23 @@ public class PaloAltoResource implements ServerResource {
                     serviceXML = "<member>any</member>";
                 }
 
-                if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { // Network egress rule
+                // handle different types of fire wall rules (egress | ingress)
+                if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) { // Egress Rule
                     srcZone = _privateZone;
                     dstZone = _publicZone;
                     dstAddressXML = "<member>any</member>";
-                } else {
+
+                    // defaults to 'allow', the deny rules are as follows
+                    if (rule.getType() == FirewallRule.FirewallRuleType.System) {
+                        if (!rule.isDefaultEgressPolicy()) { // default of deny && system rule, so deny
+                            action = "deny";
+                        }
+                    } else {
+                        if (rule.isDefaultEgressPolicy()) { // default is allow && user rule, so deny
+                            action = "deny";
+                        }
+                    }
+                } else { // Ingress Rule
                     srcZone = _publicZone;
                     dstZone = _privateZone;
                     dstAddressXML = "<member>" + rule.getSrcIp() + "</member>";
@@ -1265,6 +1303,7 @@ public class PaloAltoResource implements ServerResource {
                     }
                 }
 
+                // build new rule xml
                 String xml = "";
                 xml += "<from><member>" + srcZone + "</member></from>";
                 xml += "<to><member>" + dstZone + "</member></to>";
@@ -1272,16 +1311,52 @@ public class PaloAltoResource implements ServerResource {
                 xml += "<destination>" + dstAddressXML + "</destination>";
                 xml += "<application>" + appXML + "</application>";
                 xml += "<service>" + serviceXML + "</service>";
-                xml += "<action>allow</action>";
+                xml += "<action>"+action+"</action>";
                 xml += "<negate-source>no</negate-source>";
                 xml += "<negate-destination>no</negate-destination>";
-                if (_threatProfile != null) { // add the threat profile if it exists
+                if (_threatProfile != null && action.equals("allow")) { // add the threat profile if it exists
                     xml += "<profile-setting><group><member>" + _threatProfile + "</member></group></profile-setting>";
                 }
-                if (_logProfile != null) { // add the log profile if it exists
+                if (_logProfile != null && action.equals("allow")) { // add the log profile if it exists
                     xml += "<log-setting>" + _logProfile + "</log-setting>";
                 }
 
+                boolean has_default = false;
+                String defaultEgressRule = "";
+                if (rule.getTrafficType() == FirewallRule.TrafficType.Egress) {
+                    // check if a default egress rule exists because it always has to be after the other rules.
+                    Map<String, String> e_params = new HashMap<String, String>();
+                    e_params.put("type", "config");
+                    e_params.put("action", "get");
+                    e_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_"+rule.getSrcVlanTag()+"']");
+                    String e_response = request(PaloAltoMethod.GET, e_params);
+                    has_default = (validResponse(e_response) && responseNotEmpty(e_response));
+    
+                    // there is an existing default rule, so we need to remove it and add it back after the new rule is added.
+                    if (has_default) {
+                        s_logger.debug("Moving the default egress rule after the new rule: "+ruleName);
+                        NodeList response_body;
+                        Document doc = getDocument(e_response);
+                        XPath xpath = XPathFactory.newInstance().newXPath();
+                        try {
+                            XPathExpression expr = xpath.compile("/response[@status='success']/result/entry/node()");
+                            response_body = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);
+                        } catch (XPathExpressionException e) {
+                            throw new ExecutionException(e.getCause().getMessage());
+                        }
+                        for (int i=0; i<response_body.getLength(); i++) {
+                            Node n = response_body.item(i);
+                            defaultEgressRule += nodeToString(n);
+                        }
+                        Map<String, String> dd_params = new HashMap<String, String>();
+                        dd_params.put("type", "config");
+                        dd_params.put("action", "delete");
+                        dd_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_"+rule.getSrcVlanTag()+"']");
+                        cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, dd_params));
+                    }
+                }
+    
+                // add the new rule...
                 Map<String, String> a_params = new HashMap<String, String>();
                 a_params.put("type", "config");
                 a_params.put("action", "set");
@@ -1289,6 +1364,17 @@ public class PaloAltoResource implements ServerResource {
                 a_params.put("element", xml);
                 cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, a_params));
 
+                // add back the default rule
+                if (rule.getTrafficType() == FirewallRule.TrafficType.Egress && has_default) {
+                    Map<String, String> da_params = new HashMap<String, String>();
+                    da_params.put("type", "config");
+                    da_params.put("action", "set");
+                    da_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_"+rule.getSrcVlanTag()+"']");
+                    da_params.put("element", defaultEgressRule);
+                    cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, da_params));
+                    s_logger.debug("Completed move of the default egress rule after rule: "+ruleName);
+                }
+
                 return true;
 
             case DELETE:
@@ -1310,6 +1396,25 @@ public class PaloAltoResource implements ServerResource {
         }
     }
 
+    // remove orphaned rules if they exist...
+    public void removeOrphanedFirewallRules(ArrayList<IPaloAltoCommand> cmdList, long vlan) throws ExecutionException {
+        Map<String, String> params = new HashMap<String, String>();
+        params.put("type", "config");
+        params.put("action", "get");
+        params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[contains(@name, 'policy') and contains(@name, '"+Long.toString(vlan)+"')]");
+        String response = request(PaloAltoMethod.GET, params);
+        boolean has_orphans = (validResponse(response) && responseNotEmpty(response));
+
+        if (has_orphans) {
+            Map<String, String> d_params = new HashMap<String, String>();
+            d_params.put("type", "config");
+            d_params.put("action", "delete");
+            d_params.put("xpath", "/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[contains(@name, 'policy') and contains(@name, '"+Long.toString(vlan)+"')]");
+            cmdList.add(new DefaultPaloAltoCommand(PaloAltoMethod.POST, d_params));
+        }
+    }
+
+
     /*
      * Usage
      */
@@ -1935,6 +2040,10 @@ public class PaloAltoResource implements ServerResource {
         return ip.replace('.', '-').replace('/', '-');
     }
 
+    private String parsePublicVlanTag(String uri) {
+        return uri.replace("vlan://", "");
+    }
+
     private Protocol getProtocol(String protocolName) throws ExecutionException {
         protocolName = protocolName.toLowerCase();
 
@@ -1964,6 +2073,20 @@ public class PaloAltoResource implements ServerResource {
         }
     }
 
+    // return an xml node as a string
+    private String nodeToString(Node node) throws ExecutionException {
+        StringWriter sw = new StringWriter();
+        try {
+            Transformer t = TransformerFactory.newInstance().newTransformer();
+            t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
+            t.transform(new DOMSource(node), new StreamResult(sw));
+        } catch (Throwable t) {
+            throw new ExecutionException("XML convert error when modifying PA config: "+t.getMessage());
+        }
+        return sw.toString();
+    }
+
+    // pretty printing of xml strings
     private String prettyFormat(String input) {
         int indent = 4;
         try {
@@ -1999,14 +2122,12 @@ public class PaloAltoResource implements ServerResource {
     @Override
     public void setName(String name) {
         // TODO Auto-generated method stub
-
     }
 
     //@Override
     @Override
     public void setConfigParams(Map<String, Object> params) {
         // TODO Auto-generated method stub
-
     }
 
     //@Override
@@ -2027,7 +2148,5 @@ public class PaloAltoResource implements ServerResource {
     @Override
     public void setRunLevel(int level) {
         // TODO Auto-generated method stub
-
     }
-
 }

plugins/network-elements/palo-alto/test/com/cloud/network/resource/MockablePaloAltoResource.java

@@ -161,12 +161,12 @@ public class MockablePaloAltoResource extends PaloAltoResource {
                     }
                 }
 
-                // get egress firewall rule | has_egress_fw_rule | policy_0
-                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0']")) {
+                // get egress firewall rule | has_egress_fw_rule | policy_0_3954
+                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_3954']")) {
                     if (context.containsKey("has_egress_fw_rule") && context.get("has_egress_fw_rule").equals("true")) {
-                        response =
+                        response = 
                             "<response status=\"success\" code=\"19\"><result total-count=\"1\" count=\"1\">"
-                                + "<entry name=\"policy_0\" admin=\"admin\" time=\"2013/07/03 12:43:30\"><from admin=\"admin\" time=\"2013/07/03 12:43:30\">"
+                                + "<entry name=\"policy_0_3954\" admin=\"admin\" time=\"2013/07/03 12:43:30\"><from admin=\"admin\" time=\"2013/07/03 12:43:30\">"
                                 + "<member admin=\"admin\" time=\"2013/07/03 12:43:30\">trust</member></from><to><member>untrust</member></to><source><member>10.3.96.1/20</member>"
                                 + "</source><destination><member>any</member></destination><application><member>any</member></application><service><member>cs_tcp_80</member>"
                                 + "</service><action>allow</action><negate-source>no</negate-source><negate-destination>no</negate-destination></entry></result></response>";
@@ -190,6 +190,11 @@ public class MockablePaloAltoResource extends PaloAltoResource {
                     }
                 }
 
+                // get default egress rule | policy_0_3954
+                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[contains(@name, 'policy') and contains(@name, '3954')]")) {
+                    response = "<response status=\"success\" code=\"19\"><result/></response>";
+                }
+
                 // get destination nat rule (port forwarding) | has_dst_nat_rule
                 if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/nat/rules/entry[@name='dst_nat.192-168-80-103_9']")) {
                     if (context.containsKey("has_dst_nat_rule") && context.get("has_dst_nat_rule").equals("true")) {
@@ -292,7 +297,7 @@ public class MockablePaloAltoResource extends PaloAltoResource {
                 }
 
                 // add egress firewall rule
-                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0']")) {
+                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_3954']")) {
                     response = "<response status=\"success\" code=\"20\"><msg>command succeeded</msg></response>";
                     context.put("has_egress_fw_rule", "true");
                 }
@@ -325,7 +330,7 @@ public class MockablePaloAltoResource extends PaloAltoResource {
             // action = 'delete'
             if (params.get("action").equals("delete")) {
                 // remove egress firewall rule
-                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0']")) {
+                if (params.get("xpath").equals("/config/devices/entry/vsys/entry[@name='vsys1']/rulebase/security/rules/entry[@name='policy_0_3954']")) {
                     response = "<response status=\"success\" code=\"20\"><msg>command succeeded</msg></response>";
                     context.remove("has_egress_fw_rule");
                 }

scripts/vm/hypervisor/xenserver/cloudlog

@@ -0,0 +1,12 @@
+/var/log/cloud/vmops.log {
+    daily
+    size 1M
+    rotate 20
+}
+
+/var/log/cloud/ovstunnel.log /var/log/cloud/ovs-pvlan.log /var/log/cloud/swiftxen.log /var/log/cloud/s3xen /var/log/cloud/storageplugin {
+    daily
+    size 1M
+    rotate 2
+}
+

scripts/vm/hypervisor/xenserver/ovs-pvlan

@@ -31,7 +31,7 @@ import util
 from time import localtime as _localtime, asctime as _asctime
 
 xePath = "/opt/xensource/bin/xe"
-lib.setup_logging("/var/log/ovs-pvlan.log")
+lib.setup_logging("/var/log/cloud/ovs-pvlan.log")
 dhcpSetupPath = "/opt/cloud/bin/ovs-pvlan-dhcp-host.sh"
 vmSetupPath = "/opt/cloud/bin/ovs-pvlan-vm.sh"
 getDhcpIfacePath = "/opt/cloud/bin/ovs-get-dhcp-iface.sh"
@@ -41,9 +41,9 @@ getBridgePath = "/opt/cloud/bin/ovs-get-bridge.sh"
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### VMOPS enter  %s ####" % name)
+        logging.debug("#### VMOPS enter  %s ####" % name)
         res = fn(*v, **k)
-        util.SMlog("#### VMOPS exit  %s ####" % name)
+        logging.debug("#### VMOPS exit  %s ####" % name)
         return res
     return wrapped
 

scripts/vm/hypervisor/xenserver/ovstunnel

@@ -35,7 +35,7 @@ import util
 from time import localtime as _localtime, asctime as _asctime
 
 xePath = "/opt/xensource/bin/xe"
-lib.setup_logging("/var/log/ovstunnel.log")
+lib.setup_logging("/var/log/cloud/ovstunnel.log")
 
 
 def block_ipv6_v5(bridge):
@@ -54,9 +54,9 @@ block_ipv6_handlers = {
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### VMOPS enter  %s ####" % name)
+        logging.debug("#### VMOPS enter  %s ####" % name)
         res = fn(*v, **k)
-        util.SMlog("#### VMOPS exit  %s ####" % name)
+        logging.debug("#### VMOPS exit  %s ####" % name)
         return res
     return wrapped
 

scripts/vm/hypervisor/xenserver/perfmon.py

@@ -0,0 +1,261 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pprint
+import XenAPI
+import urllib
+from xml.dom import minidom
+import time
+import commands
+
+# Per VM dictionary (used by RRDUpdates to look up column numbers by variable names)
+class VMReport(dict):
+    """Used internally by RRDUpdates"""
+    def __init__(self, uuid):
+        self.uuid = uuid
+        super(dict, self).__init__()
+
+
+# Per Host dictionary (used by RRDUpdates to look up column numbers by variable names)
+class HostReport(dict):
+    """Used internally by RRDUpdates"""
+    def __init__(self, uuid):
+        self.uuid = uuid
+        super(dict, self).__init__()
+
+
+class PerfMonException(Exception):
+    pass
+
+
+class XmlConfigException(PerfMonException):
+    pass
+
+
+class UsageException(Exception):
+    pass
+
+
+class RRDUpdates:
+    """ Object used to get and parse the output the http://localhost/rrd_udpates?...
+    """
+    def __init__(self):
+        # params are what get passed to the CGI executable in the URL
+        self.params = dict()
+        self.params['start'] = int(time.time()) - 1000  # For demo purposes!
+        self.params['host'] = 'false'   # include data for host (as well as for VMs)
+        self.params['cf'] = 'AVERAGE'  # consolidation function, each sample averages 12 from the 5 second RRD
+        self.params['interval'] = '60'
+
+    def get_nrows(self):
+        return self.rows
+
+    def get_vm_list(self):
+        return self.vm_reports.keys()
+
+    def get_vm_param_list(self, uuid):
+        report = self.vm_reports[uuid]
+        if not report:
+            return []
+        return report.keys()
+
+    def get_total_cpu_core(self, uuid):
+	report = self.vm_reports[uuid]
+        if not report:
+            return 0
+        else:
+            param_keys = report.keys()
+            result = 0
+            for param in param_keys:
+                if "cpu" in param:
+                    result += 1
+            return result
+
+    def get_vm_data(self, uuid, param, row):
+	#pp = pprint.PrettyPrinter(indent=4) 
+	#pp.pprint(self.vm_reports)
+        report = self.vm_reports[uuid]
+        col = report[param]
+        return self.__lookup_data(col, row)
+
+    def get_host_uuid(self):
+        report = self.host_report
+        if not report:
+            return None
+        return report.uuid
+
+    def get_host_param_list(self):
+        report = self.host_report
+        if not report:
+            return []
+        return report.keys()
+
+    def get_host_data(self, param, row):
+        report = self.host_report
+        col = report[param]
+        return self.__lookup_data(col, row)
+
+    def get_row_time(self, row):
+        return self.__lookup_timestamp(row)
+
+    # extract float from value (<v>) node by col,row
+    def __lookup_data(self, col, row):
+        # Note: the <rows> nodes are in reverse chronological order, and comprise
+        # a timestamp <t> node, followed by self.columns data <v> nodes
+        node = self.data_node.childNodes[self.rows - 1 - row].childNodes[col + 1]
+        return float(node.firstChild.toxml())  # node.firstChild should have nodeType TEXT_NODE
+
+    # extract int from value (<t>) node by row
+    def __lookup_timestamp(self, row):
+        # Note: the <rows> nodes are in reverse chronological order, and comprise
+        # a timestamp <t> node, followed by self.columns data <v> nodes
+        node = self.data_node.childNodes[self.rows - 1 - row].childNodes[0]
+        return int(node.firstChild.toxml())  # node.firstChild should have nodeType TEXT_NODE
+
+    def refresh(self, login, starttime, session, override_params):
+        self.params['start'] = starttime
+        params = override_params
+        params['session_id'] = session
+        params.update(self.params)
+        paramstr = "&".join(["%s=%s" % (k, params[k]) for k in params])
+        # this is better than urllib.urlopen() as it raises an Exception on http 401 'Unauthorised' error
+        # rather than drop into interactive mode
+	for host in login.host.get_all():
+		#print "http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr
+        	sock = urllib.URLopener().open("http://" + str(login.host.get_address(host)) + "/rrd_updates?%s" % paramstr)
+	        xmlsource = sock.read()
+        	sock.close()
+        	xmldoc = minidom.parseString(xmlsource)
+        	self.__parse_xmldoc(xmldoc)
+        	# Update the time used on the next run
+        	self.params['start'] = self.end_time + 1  # avoid retrieving same data twice
+
+    def __parse_xmldoc(self, xmldoc):
+        # The 1st node contains meta data (description of the data)
+        # The 2nd node contains the data
+        self.meta_node = xmldoc.firstChild.childNodes[0]
+        self.data_node = xmldoc.firstChild.childNodes[1]
+
+        def lookup_metadata_bytag(name):
+            return int(self.meta_node.getElementsByTagName(name)[0].firstChild.toxml())
+            # rows = number of samples per variable
+        # columns = number of variables
+        self.rows = lookup_metadata_bytag('rows')
+        self.columns = lookup_metadata_bytag('columns')
+        # These indicate the period covered by the data
+        self.start_time = lookup_metadata_bytag('start')
+        self.step_time = lookup_metadata_bytag('step')
+        self.end_time = lookup_metadata_bytag('end')
+        # the <legend> Node describes the variables
+        self.legend = self.meta_node.getElementsByTagName('legend')[0]
+        # vm_reports matches uuid to per VM report
+        if not hasattr(self,'vm_reports'):
+		self.vm_reports = {}
+        # There is just one host_report and its uuid should not change!
+        self.host_report = None
+        # Handle each column.  (I.e. each variable)
+        for col in range(self.columns):
+            self.__handle_col(col)
+
+    def __handle_col(self, col):
+        # work out how to interpret col from the legend
+        col_meta_data = self.legend.childNodes[col].firstChild.toxml()
+        # vm_or_host will be 'vm' or 'host'.  Note that the Control domain counts as a VM!
+        (cf, vm_or_host, uuid, param) = col_meta_data.split(':')
+        if vm_or_host == 'vm':
+            # Create a report for this VM if it doesn't exist
+            if not uuid in self.vm_reports:
+                self.vm_reports[uuid] = VMReport(uuid)
+                # Update the VMReport with the col data and meta data
+            vm_report = self.vm_reports[uuid]
+            vm_report[param] = col
+        elif vm_or_host == 'host':
+            # Create a report for the host if it doesn't exist
+            if not self.host_report:
+                self.host_report = HostReport(uuid)
+            elif self.host_report.uuid != uuid:
+                raise PerfMonException("Host UUID changed: (was %s, is %s)" % (self.host_report.uuid, uuid))
+                # Update the HostReport with the col data and meta data
+            self.host_report[param] = col
+        else:
+            raise PerfMonException("Invalid string in <legend>: %s" % col_meta_data)
+
+def getuuid(vm_name):
+    status, output = commands.getstatusoutput("xe vm-list | grep "+vm_name+" -B 1 | head -n 1 | awk -F':' '{print $2}' | tr -d ' '")
+    if (status != 0):
+	raise PerfMonException("Invalid vm name: %s" % vm_name)
+    return output
+
+def get_vm_group_perfmon(args={}):
+    login = XenAPI.xapi_local()
+    login.login_with_password("","")
+    result = ""
+
+    total_vm = int(args['total_vm'])
+    total_counter = int(args['total_counter'])
+    now = int(time.time()) / 60
+
+    # Get pool's info of this host
+    #pool = login.xenapi.pool.get_all()[0]
+    # Get master node's address of pool
+    #master = login.xenapi.pool.get_master(pool)
+    #master_address = login.xenapi.host.get_address(master)
+    session = login._session
+
+    max_duration = 0
+    for counter_count in xrange(1, total_counter + 1):
+        duration = int(args['duration' + str(counter_count)])
+        if duration > max_duration:
+            max_duration = duration
+
+    rrd_updates = RRDUpdates()
+    rrd_updates.refresh(login.xenapi, now * 60 - max_duration, session, {})
+
+    #for uuid in rrd_updates.get_vm_list():
+    for vm_count in xrange(1, total_vm + 1):
+	vm_name = args['vmname' + str(vm_count)]
+        vm_uuid = getuuid(vm_name)
+        #print "Got values for VM: " + str(vm_count) + " " + vm_uuid
+        for counter_count in xrange(1, total_counter + 1):
+	    #refresh average
+	    average_cpu = 0
+	    average_memory = 0
+            counter = args['counter' + str(counter_count)]
+            total_row = rrd_updates.get_nrows()
+            duration = int(args['duration' + str(counter_count)]) / 60
+            duration_diff = total_row - duration
+            if counter == "cpu":
+		total_cpu = rrd_updates.get_total_cpu_core(vm_uuid)
+                for row in xrange(duration_diff, total_row):
+                    for cpu in xrange(0, total_cpu):
+                        average_cpu += rrd_updates.get_vm_data(vm_uuid, "cpu" + str(cpu), row)
+                average_cpu /= (duration * total_cpu)
+                if result == "":
+                    result += str(vm_count) + '.' +  str(counter_count) + ':' + str(average_cpu)
+                else:
+                    result += ',' + str(vm_count) +  '.' + str(counter_count) + ':' + str(average_cpu)
+            elif counter == "memory":
+                for row in xrange(duration_diff, total_row):
+                    average_memory += rrd_updates.get_vm_data(vm_uuid, "memory_target", row) / 1048576 - rrd_updates.get_vm_data(vm_uuid, "memory_internal_free", row) / 1024
+                average_memory /= duration
+                if result == "":
+                    result += str(vm_count) +  '.' +  str(counter_count) + ':' + str(average_memory)
+                else:
+                    result += ',' + str(vm_count) +  '.' +  str(counter_count) + ':' + str(average_memory)
+    return result
+

scripts/vm/hypervisor/xenserver/s3xen

@@ -39,6 +39,10 @@ from xml.dom.minidom import parseString
 import XenAPIPlugin
 sys.path.extend(["/opt/xensource/sm/"])
 import util
+import cloudstack_pluginlib as lib
+import logging
+
+lib.setup_logging("/var/log/cloud/s3xen.log")
 
 NULL = 'null'
 
@@ -102,7 +106,7 @@ def get_optional_key(map, key, default=''):
 
 def log(message):
 
-    util.SMlog('#### VMOPS %s ####' % message)
+    logging.debug('#### VMOPS %s ####' % message)
 
 
 def echo(fn):

scripts/vm/hypervisor/xenserver/storagePlugin

@@ -32,13 +32,17 @@ import subprocess
 import zlib
 import urllib2
 import traceback
+import cloudstack_pluginlib as lib
+import logging
+
+lib.setup_logging("/var/log/cloud/storageplugin.log")
 
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### xen plugin enter  %s ####" % name )
+        logging.debug("#### xen plugin enter  %s ####" % name )
         res = fn(*v, **k)
-        util.SMlog("#### xen plugin exit  %s ####" % name )
+        logging.debug("#### xen plugin exit  %s ####" % name )
         return res
     return wrapped
 
@@ -53,7 +57,7 @@ def downloadTemplateFromUrl(session, args):
         destFile.close()
         return "success"
     except:
-        util.SMlog("exception: " + str(sys.exc_info()))
+        logging.debug("exception: " + str(sys.exc_info()))
         return ""
     
 @echo

scripts/vm/hypervisor/xenserver/swiftxen

@@ -24,13 +24,17 @@ import os, sys, time
 import XenAPIPlugin
 sys.path.extend(["/opt/xensource/sm/"])
 import util
+import cloudstack_pluginlib as lib
+import logging
+
+lib.setup_logging("/var/log/cloud/swiftxen.log")
 
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### VMOPS enter  %s ####" % name )
+        logging.debug("#### VMOPS enter  %s ####" % name )
         res = fn(*v, **k)
-        util.SMlog("#### VMOPS exit  %s ####" % name )
+        logging.debug("#### VMOPS exit  %s ####" % name )
         return res
     return wrapped
 
@@ -48,7 +52,7 @@ def upload(args):
     lfilename = args['lfilename']
     isISCSI = args['isISCSI']
     segment = 0
-    util.SMlog("#### VMOPS upload %s to swift ####", lfilename)
+    logging.debug("#### VMOPS upload %s to swift ####", lfilename)
     savedpath = os.getcwd()
     os.chdir(ldir)
     try :
@@ -85,7 +89,7 @@ def swift(session, args):
     elif op == 'delete' :
         cmd = ["st", "-A https://" + hostname + ":8080/auth/v1.0 -U " + account + ":" + username + " -K " + token + " delete " + rfilename]
     else :
-        util.SMlog("doesn't support swift operation  %s " % op )
+        logging.debug("doesn't support swift operation  %s " % op )
         return 'false'
     try:
         util.pread2(cmd)

scripts/vm/hypervisor/xenserver/vmops

@@ -34,14 +34,18 @@ import tempfile
 import util
 import subprocess
 import zlib
+import cloudstack_pluginlib as lib
+import logging
 from util import CommandException
 
+lib.setup_logging("/var/log/cloud/vmops.log")
+
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### VMOPS enter  %s ####" % name )
+        logging.debug("#### VMOPS enter  %s ####" % name )
         res = fn(*v, **k)
-        util.SMlog("#### VMOPS exit  %s ####" % name )
+        logging.debug("#### VMOPS exit  %s ####" % name )
         return res
     return wrapped
 
@@ -96,7 +100,7 @@ def preparemigration(session, args):
         util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog("Catch prepare migration exception" )
+        logging.debug("Catch prepare migration exception" )
         txt = ''
 
     return txt
@@ -108,7 +112,7 @@ def setIptables(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog("  setIptables execution failed "  )
+        logging.debug("  setIptables execution failed "  )
         txt = '' 
 
     return txt
@@ -153,7 +157,7 @@ def pingtest(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog("  pingtest failed "  )
+        logging.debug("  pingtest failed "  )
         txt = ''
 
     return txt
@@ -168,7 +172,7 @@ def savePassword(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog("  save password to domr failed "  )
+        logging.debug("  save password to domr failed "  )
         txt = '' 
 
     return txt
@@ -183,7 +187,7 @@ def saveDhcpEntry(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog(" save dhcp entry failed "  )
+        logging.debug(" save dhcp entry failed "  )
         txt = '' 
 
     return txt
@@ -245,7 +249,7 @@ def setFirewallRule(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog(" set firewall rule failed "  )
+        logging.debug(" set firewall rule failed "  )
         txt = '' 
 
     return txt
@@ -261,7 +265,7 @@ def routerProxy(session, args):
         if txt is None or len(txt) == 0 :
             txt = 'success'
     except:
-        util.SMlog("routerProxy command " + sargs + " failed "  )
+        logging.debug("routerProxy command " + sargs + " failed "  )
         txt = '' 
 
     return txt
@@ -278,7 +282,7 @@ def setLoadBalancerRule(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog(" set loadbalancer rule failed "  )
+        logging.debug(" set loadbalancer rule failed "  )
         txt = '' 
 
     return txt
@@ -291,7 +295,7 @@ def configdnsmasq(session, args):
        util.pread2(['ssh','-p','3922','-q','-o','StrictHostKeyChecking=no','-i','/root/.ssh/id_rsa.cloud',target,'/root/dnsmasq.sh',args])
        txt='success'
     except:
-       util.SMlog("failed to config dnsmasq server")
+       logging.debug("failed to config dnsmasq server")
        txt=''
     return txt
 
@@ -305,7 +309,7 @@ def createipAlias(session, args):
        txt=util.pread2(cmd)
        txt='success'
     except:
-       util.SMlog("failed to create ip alias on router vm")
+       logging.debug("failed to create ip alias on router vm")
        txt=''
     return txt
 
@@ -319,7 +323,7 @@ def deleteipAlias(session, args):
        txt=util.pread2(cmd)
        txt='success'
     except:
-       util.SMlog("failed to create ip alias on router vm")
+       logging.debug("failed to create ip alias on router vm")
        txt=''
     return txt
 
@@ -334,7 +338,7 @@ def createFile(session, args):
         f.close()
         txt = 'success'
     except:
-        util.SMlog(" failed to create HA proxy cfg file ")
+        logging.debug(" failed to create HA proxy cfg file ")
         txt = ''
 
     return txt
@@ -354,7 +358,7 @@ def createFileInDomr(session, args):
         util.pread2(['rm',tmpfile])
         txt = 'success'
     except:
-        util.SMlog(" failed to create HA proxy cfg file ")
+        logging.debug(" failed to create HA proxy cfg file ")
         txt = ''
 
     return txt
@@ -368,7 +372,7 @@ def deleteFile(session, args):
             os.remove(file_path)
         txt = 'success'
     except:
-        util.SMlog(" failed to remove HA proxy cfg file ")
+        logging.debug(" failed to remove HA proxy cfg file ")
         txt = ''
 
     return txt
@@ -409,7 +413,7 @@ def can_bridge_firewall(session, args):
         util.pread2(['iptables', '-A', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged',  '-p', 'udp', '--dport', '68', '--sport', '67',  '-j', 'ACCEPT'])
         util.pread2(['iptables', '-D', 'FORWARD',  '-j', 'RH-Firewall-1-INPUT'])
     except:
-        util.SMlog('Chain BRIDGE-FIREWALL already exists')
+        logging.debug('Chain BRIDGE-FIREWALL already exists')
 
     try:
         util.pread2(['iptables', '-N', 'BRIDGE-DEFAULT-FIREWALL'])
@@ -421,7 +425,7 @@ def can_bridge_firewall(session, args):
         util.pread2(['iptables', '-D', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged',  '-p', 'udp', '--dport', '67', '--sport', '68',  '-j', 'ACCEPT'])
         util.pread2(['iptables', '-D', 'BRIDGE-FIREWALL', '-m', 'physdev', '--physdev-is-bridged',  '-p', 'udp', '--dport', '68', '--sport', '67',  '-j', 'ACCEPT'])
     except:
-        util.SMlog('Chain BRIDGE-DEFAULT-FIREWALL already exists')
+        logging.debug('Chain BRIDGE-DEFAULT-FIREWALL already exists')
 
     result = 'true'
     try:
@@ -468,7 +472,7 @@ def default_ebtables_rules():
         # deny all others (e.g., 802.1d, CDP)
         util.pread2(['ebtables', '-A', 'DEFAULT_EBTABLES',  '-j', 'DROP'])
     except:
-        util.SMlog('Chain DEFAULT_EBTABLES already exists')
+        logging.debug('Chain DEFAULT_EBTABLES already exists')
 
 
 @echo
@@ -485,7 +489,7 @@ def allow_egress_traffic(session):
             try:
                 util.pread2(['iptables', '-I', 'FORWARD', '2', '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', d, '-j', 'ACCEPT'])
             except:
-                util.SMlog("Failed to add FORWARD rule through to %s" % d)
+                logging.debug("Failed to add FORWARD rule through to %s" % d)
                 return 'false'
     return 'true'
 
@@ -494,7 +498,7 @@ def ipset(ipsetname, proto, start, end, ips):
     try:
         util.pread2(['ipset', '-N', ipsetname, 'iptreemap'])
     except:
-        util.SMlog("ipset chain already exists" + ipsetname)
+        logging.debug("ipset chain already exists" + ipsetname)
 
     result = True
     ipsettmp = ''.join(''.join(ipsetname.split('-')).split('_')) + str(int(time.time()) % 1000)
@@ -502,11 +506,11 @@ def ipset(ipsetname, proto, start, end, ips):
     try: 
         util.pread2(['ipset', '-N', ipsettmp, 'iptreemap']) 
     except:
-        util.SMlog("Failed to create temp ipset, reusing old name= " + ipsettmp)
+        logging.debug("Failed to create temp ipset, reusing old name= " + ipsettmp)
         try: 
             util.pread2(['ipset', '-F', ipsettmp]) 
         except:
-            util.SMlog("Failed to clear old temp ipset name=" + ipsettmp)
+            logging.debug("Failed to clear old temp ipset name=" + ipsettmp)
             return False
         
     try: 
@@ -517,7 +521,7 @@ def ipset(ipsetname, proto, start, end, ips):
                 if cex.reason.rfind('already in set') == -1:
                    raise
     except:
-        util.SMlog("Failed to program ipset " + ipsetname)
+        logging.debug("Failed to program ipset " + ipsetname)
         util.pread2(['ipset', '-F', ipsettmp]) 
         util.pread2(['ipset', '-X', ipsettmp]) 
         return False
@@ -525,7 +529,7 @@ def ipset(ipsetname, proto, start, end, ips):
     try: 
         util.pread2(['ipset', '-W', ipsettmp, ipsetname]) 
     except:
-        util.SMlog("Failed to swap ipset " + ipsetname)
+        logging.debug("Failed to swap ipset " + ipsetname)
         result = False
 
     try: 
@@ -533,7 +537,7 @@ def ipset(ipsetname, proto, start, end, ips):
         util.pread2(['ipset', '-X', ipsettmp]) 
     except:
         # if the temporary name clashes next time we'll just reuse it
-        util.SMlog("Failed to delete temp ipset " + ipsettmp)
+        logging.debug("Failed to delete temp ipset " + ipsettmp)
 
     return result
 
@@ -550,7 +554,7 @@ def destroy_network_rules_for_vm(session, args):
             util.pread2(['iptables', '-F', vmchain_default])
             util.pread2(['iptables', '-X', vmchain_default])
         except:
-            util.SMlog("Ignoring failure to delete  chain " + vmchain_default)
+            logging.debug("Ignoring failure to delete  chain " + vmchain_default)
     
     destroy_ebtables_rules(vmchain)
     destroy_arptables_rules(vmchain)
@@ -559,14 +563,14 @@ def destroy_network_rules_for_vm(session, args):
         util.pread2(['iptables', '-F', vmchain])
         util.pread2(['iptables', '-X', vmchain])
     except:
-        util.SMlog("Ignoring failure to delete ingress chain " + vmchain)
+        logging.debug("Ignoring failure to delete ingress chain " + vmchain)
         
    
     try:
         util.pread2(['iptables', '-F', vmchain_egress])
         util.pread2(['iptables', '-X', vmchain_egress])
     except:
-        util.SMlog("Ignoring failure to delete egress chain " + vmchain_egress)
+        logging.debug("Ignoring failure to delete egress chain " + vmchain_egress)
     
     remove_rule_log_for_vm(vm_name)
     remove_secip_log_for_vm(vm_name)
@@ -582,7 +586,7 @@ def destroy_network_rules_for_vm(session, args):
                 util.pread2(['ipset', '-F', set])       
                 util.pread2(['ipset', '-X', set])       
     except:
-        util.SMlog("Failed to destroy ipsets for %" % vm_name)
+        logging.debug("Failed to destroy ipsets for %" % vm_name)
     
     
     return 'true'
@@ -599,12 +603,12 @@ def destroy_ebtables_rules(vm_chain):
             dc.insert(0, 'ebtables')
             util.pread2(dc)
         except:
-            util.SMlog("Ignoring failure to delete ebtables rules for vm " + vm_chain)
+            logging.debug("Ignoring failure to delete ebtables rules for vm " + vm_chain)
     try:
         util.pread2(['ebtables', '-F', vm_chain])
         util.pread2(['ebtables', '-X', vm_chain])
     except:
-            util.SMlog("Ignoring failure to delete ebtables chain for vm " + vm_chain)   
+            logging.debug("Ignoring failure to delete ebtables chain for vm " + vm_chain)   
 
 @echo
 def destroy_arptables_rules(vm_chain):
@@ -619,18 +623,18 @@ def destroy_arptables_rules(vm_chain):
             dc.insert(2, 'FORWARD')
             util.pread2(dc)
         except:
-            util.SMlog("Ignoring failure to delete arptables rules for vm " + vm_chain)
+            logging.debug("Ignoring failure to delete arptables rules for vm " + vm_chain)
     
     try:
         util.pread2(['arptables', '-F', vm_chain])
         util.pread2(['arptables', '-X', vm_chain])
     except:
-        util.SMlog("Ignoring failure to delete arptables chain for vm " + vm_chain) 
+        logging.debug("Ignoring failure to delete arptables chain for vm " + vm_chain) 
               
 @echo
 def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac):
     if vm_mac == 'ff:ff:ff:ff:ff:ff':
-        util.SMlog("Ignoring since mac address is not valid")
+        logging.debug("Ignoring since mac address is not valid")
         return 'true'
     
     try:
@@ -639,7 +643,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac):
         try:
             util.pread2(['ebtables', '-F', vm_chain])
         except:
-            util.SMlog("Failed to create ebtables antispoof chain, skipping")
+            logging.debug("Failed to create ebtables antispoof chain, skipping")
             return 'true'
 
     # note all rules for packets into the bridge (-i) precede all output rules (-o)
@@ -649,7 +653,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac):
             util.pread2(['ebtables', '-I', 'FORWARD', '2', '-i',  vif,  '-j', vm_chain])
             util.pread2(['ebtables', '-A', 'FORWARD', '-o',  vif, '-j', vm_chain])
     except:
-        util.SMlog("Failed to program default ebtables FORWARD rules for %s" % vm_chain)
+        logging.debug("Failed to program default ebtables FORWARD rules for %s" % vm_chain)
         return 'false'
 
     try:
@@ -665,7 +669,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac):
             # do not allow snooping of dhcp requests
             util.pread2(['ebtables', '-A', vm_chain, '-o', vif, '-p', 'IPv4', '--ip-proto', 'udp', '--ip-dport', '67', '-j', 'DROP'])
     except:
-        util.SMlog("Failed to program default ebtables antispoof rules for %s" % vm_chain)
+        logging.debug("Failed to program default ebtables antispoof rules for %s" % vm_chain)
         return 'false'
 
     return 'true'
@@ -673,7 +677,7 @@ def default_ebtables_antispoof_rules(vm_chain, vifs, vm_ip, vm_mac):
 @echo
 def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac):
     if vm_mac == 'ff:ff:ff:ff:ff:ff':
-        util.SMlog("Ignoring since mac address is not valid")
+        logging.debug("Ignoring since mac address is not valid")
         return 'true'
 
     try:
@@ -682,7 +686,7 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac):
         try:
             util.pread2(['arptables', '-F', vm_chain])
         except:
-            util.SMlog("Failed to create arptables rule, skipping")
+            logging.debug("Failed to create arptables rule, skipping")
             return 'true'
 
     # note all rules for packets into the bridge (-i) precede all output rules (-o)
@@ -691,7 +695,7 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac):
            util.pread2(['arptables',  '-I', 'FORWARD', '-i',  vif, '-j', vm_chain])
            util.pread2(['arptables',  '-A', 'FORWARD', '-o',  vif, '-j', vm_chain])
     except:
-        util.SMlog("Failed to program default arptables rules in FORWARD chain vm=" + vm_chain)
+        logging.debug("Failed to program default arptables rules in FORWARD chain vm=" + vm_chain)
         return 'false'
     
     try:
@@ -708,7 +712,7 @@ def default_arp_antispoof(vm_chain, vifs, vm_ip, vm_mac):
         util.pread2(['arptables',  '-A', vm_chain,  '-j', 'DROP'])
 
     except:
-        util.SMlog("Failed to program default arptables  rules")
+        logging.debug("Failed to program default arptables  rules")
         return 'false'
 
     return 'true'
@@ -720,10 +724,10 @@ def network_rules_vmSecondaryIp(session, args):
     vm_mac = args.pop('vmMac')
     ip_secondary = args.pop('vmSecIp')
     action = args.pop('action')
-    util.SMlog("vmMac = "+ vm_mac)
-    util.SMlog("vmName = "+ vm_name)
+    logging.debug("vmMac = "+ vm_mac)
+    logging.debug("vmName = "+ vm_name)
     #action = "-A"
-    util.SMlog("action = "+ action)
+    logging.debug("action = "+ action)
     try:
         vm = session.xenapi.VM.get_by_name_label(vm_name)
         if len(vm) != 1:
@@ -733,11 +737,11 @@ def network_rules_vmSecondaryIp(session, args):
         vifnums = [session.xenapi.VIF.get_record(vif).get('device') for vif in vm_vifs]
         domid = vm_rec.get('domid')
     except:
-        util.SMlog("### Failed to get domid or vif list for vm  ##" + vm_name)
+        logging.debug("### Failed to get domid or vif list for vm  ##" + vm_name)
         return 'false'
 
     if domid == '-1':
-        util.SMlog("### Failed to get domid for vm (-1):  " + vm_name)
+        logging.debug("### Failed to get domid for vm (-1):  " + vm_name)
         return 'false'
 
     vifs = ["vif" + domid + "." + v for v in vifnums]
@@ -767,11 +771,11 @@ def default_network_rules_systemvm(session, args):
         vifnums = [session.xenapi.VIF.get_record(vif).get('device') for vif in vm_vifs]
         domid = vm_rec.get('domid')
     except:
-        util.SMlog("### Failed to get domid or vif list for vm  ##" + vm_name)
+        logging.debug("### Failed to get domid or vif list for vm  ##" + vm_name)
         return 'false'
     
     if domid == '-1':
-        util.SMlog("### Failed to get domid for vm (-1):  " + vm_name)
+        logging.debug("### Failed to get domid for vm (-1):  " + vm_name)
         return 'false'
 
     vifs = ["vif" + domid + "." + v for v in vifnums]
@@ -792,26 +796,26 @@ def default_network_rules_systemvm(session, args):
             util.pread2(['iptables', '-I', 'BRIDGE-FIREWALL', '2', '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', vif, '-j', vmchain])
             util.pread2(['iptables', '-I', vmchain, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', vif, '-j', 'RETURN'])
         except:
-            util.SMlog("Failed to program default rules")
+            logging.debug("Failed to program default rules")
             return 'false'
 	
 	
     util.pread2(['iptables', '-A', vmchain, '-j', 'ACCEPT'])
     
     if write_rule_log_for_vm(vm_name, '-1', '_ignore_', domid, '_initial_', '-1') == False:
-        util.SMlog("Failed to log default network rules for systemvm, ignoring")
+        logging.debug("Failed to log default network rules for systemvm, ignoring")
     return 'true'
 
 @echo
 def create_ipset_forvm (ipsetname):
     result = True
     try:
-        util.SMlog("Creating ipset chain .... " + ipsetname)
+        logging.debug("Creating ipset chain .... " + ipsetname)
         util.pread2(['ipset', '-F', ipsetname])
         util.pread2(['ipset', '-X', ipsetname])
         util.pread2(['ipset', '-N', ipsetname, 'iphash'])
     except:
-        util.SMlog("ipset chain not exists creating.... " + ipsetname)
+        logging.debug("ipset chain not exists creating.... " + ipsetname)
         util.pread2(['ipset', '-N', ipsetname, 'iphash'])
 
     return result
@@ -821,10 +825,10 @@ def add_to_ipset(ipsetname, ips, action):
     result = True
     for ip in ips:
         try:
-            util.SMlog("vm ip " + ip)
+            logging.debug("vm ip " + ip)
             util.pread2(['ipset', action, ipsetname, ip])
         except:
-            util.SMlog("vm ip alreday in ip set" + ip)
+            logging.debug("vm ip alreday in ip set" + ip)
             continue
 
     return result
@@ -846,7 +850,7 @@ def arp_rules_vmip (vm_chain, vifs, ips, vm_mac, action):
                 #also important to restrict source ip and src mac in these requests as they can be used to update arp tables on destination
                 util.pread2(['arptables',  action, vm_chain, '-i', vif, '--opcode', 'Request',  '--source-mac',  vm_mac, '--source-ip',  vm_ip, '-j', 'RETURN'])
     except:
-        util.SMlog("Failed to program arptables  rules for ip")
+        logging.debug("Failed to program arptables  rules for ip")
         return 'false'
 
     return 'true'
@@ -864,15 +868,15 @@ def default_network_rules(session, args):
     try:
         vm = session.xenapi.VM.get_by_name_label(vm_name)
         if len(vm) != 1:
-             util.SMlog("### Failed to get record for vm  " + vm_name)
+             logging.debug("### Failed to get record for vm  " + vm_name)
              return 'false'
         vm_rec = session.xenapi.VM.get_record(vm[0])
         domid = vm_rec.get('domid')
     except:
-        util.SMlog("### Failed to get domid for vm " + vm_name)
+        logging.debug("### Failed to get domid for vm " + vm_name)
         return 'false'
     if domid == '-1':     
-        util.SMlog("### Failed to get domid for vm (-1):  " + vm_name)
+        logging.debug("### Failed to get domid for vm (-1):  " + vm_name)
         return 'false'
     
     vif = "vif" + domid + ".0"
@@ -912,12 +916,12 @@ def default_network_rules(session, args):
     vmipset = vm_name
     #create ipset and add vm ips to that ip set
     if create_ipset_forvm(vmipset) == False:
-       util.SMlog(" failed to create ipset for rule " + str(tokens))
+       logging.debug(" failed to create ipset for rule " + str(tokens))
        return 'false'
 
     #add primary nic ip to ipset
     if add_to_ipset(vmipset, [vm_ip], action ) == False:
-       util.SMlog(" failed to add vm " + vm_ip + " ip to set ")
+       logging.debug(" failed to add vm " + vm_ip + " ip to set ")
        return 'false'
 
     #add secodnary nic ips to ipset
@@ -928,10 +932,10 @@ def default_network_rules(session, args):
         secIpSet = "0";
 
     if secIpSet == "1":
-        util.SMlog("Adding ipset for secondary ips")
+        logging.debug("Adding ipset for secondary ips")
         add_to_ipset(vmipset, ips, action)
         if write_secip_log_for_vm(vm_name, sec_ips, vm_id) == False:
-            util.SMlog("Failed to log default network rules, ignoring")
+            logging.debug("Failed to log default network rules, ignoring")
 
     keyword = '--' + get_ipset_keyword()
 
@@ -949,21 +953,21 @@ def default_network_rules(session, args):
             util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-in', v, '-m', 'set', keyword, vmipset, 'src', '-j', vmchain_egress])
             util.pread2(['iptables', '-A', vmchain_default, '-m', 'physdev', '--physdev-is-bridged', '--physdev-out', v,  '-j', vmchain])
     except:
-        util.SMlog("Failed to program default rules for vm " + vm_name)
+        logging.debug("Failed to program default rules for vm " + vm_name)
         return 'false'
     
     default_arp_antispoof(vmchain, vifs, vm_ip, vm_mac)
     #add default arp rules for secondary ips;
     if secIpSet == "1":
-        util.SMlog("Adding arp rules for sec ip")
+        logging.debug("Adding arp rules for sec ip")
         arp_rules_vmip(vmchain, vifs, ips, vm_mac, action)
 
     default_ebtables_antispoof_rules(vmchain, vifs, vm_ip, vm_mac)
     
     if write_rule_log_for_vm(vm_name, vm_id, vm_ip, domid, '_initial_', '-1', vm_mac) == False:
-        util.SMlog("Failed to log default network rules, ignoring")
+        logging.debug("Failed to log default network rules, ignoring")
         
-    util.SMlog("Programmed default rules for vm " + vm_name)
+    logging.debug("Programmed default rules for vm " + vm_name)
     return 'true'
 
 @echo
@@ -972,12 +976,12 @@ def check_domid_changed(session, vmName):
     try:
         vm = session.xenapi.VM.get_by_name_label(vmName)
         if len(vm) != 1:
-             util.SMlog("### Could not get record for vm ## " + vmName)
+             logging.debug("### Could not get record for vm ## " + vmName)
         else:
             vm_rec = session.xenapi.VM.get_record(vm[0])
             curr_domid = vm_rec.get('domid')
     except:
-        util.SMlog("### Failed to get domid for vm  ## " + vmName)
+        logging.debug("### Failed to get domid for vm  ## " + vmName)
         
     
     logfilename = "/var/run/cloud/" + vmName +".log"
@@ -1011,7 +1015,7 @@ def delete_rules_for_vm_in_bridge_firewall_chain(vmName):
             dc.pop()
             util.pread2(filter(None, dc))
         except:
-              util.SMlog("Ignoring failure to delete rules for vm " + vmName)
+              logging.debug("Ignoring failure to delete rules for vm " + vmName)
 
   
 @echo
@@ -1028,7 +1032,7 @@ def network_rules_for_rebooted_vm(session, vmName):
     if curr_domid == '-1':
         return True
     
-    util.SMlog("Found a rebooted VM -- reprogramming rules for  " + vm_name)
+    logging.debug("Found a rebooted VM -- reprogramming rules for  " + vm_name)
     
     delete_rules_for_vm_in_bridge_firewall_chain(vm_name)
     if 1 in [ vm_name.startswith(c) for c in ['r-', 's-', 'v-', 'l-'] ]:
@@ -1073,9 +1077,9 @@ def network_rules_for_rebooted_vm(session, vmName):
             try:
                 util.pread2(filter(None,ipt))
             except:
-                util.SMlog("Failed to rewrite antispoofing rules for vm " + vm_name)
+                logging.debug("Failed to rewrite antispoofing rules for vm " + vm_name)
     except:
-        util.SMlog("No rules found for vm " + vm_name)
+        logging.debug("No rules found for vm " + vm_name)
 
     destroy_ebtables_rules(vmchain)
     destroy_arptables_rules(vmchain)
@@ -1179,7 +1183,7 @@ def get_rule_logs_for_vms(session, args):
         hostrec = session.xenapi.host.get_record(thishost)
         vms = hostrec.get('resident_VMs')
     except:
-        util.SMlog("Failed to get host from uuid " + host_uuid)
+        logging.debug("Failed to get host from uuid " + host_uuid)
         return ' '
     
     result = []
@@ -1192,7 +1196,7 @@ def get_rule_logs_for_vms(session, args):
                 log = get_rule_log_for_vm(session, name)
                 result.append(log)
     except:
-        util.SMlog("Failed to get rule logs, better luck next time!")
+        logging.debug("Failed to get rule logs, better luck next time!")
         
     return ";".join(result)
 
@@ -1209,13 +1213,13 @@ def cleanup_rules_for_dead_vms(session):
             vm_rec = session.xenapi.VM.get_record(vm[0])
             state = vm_rec.get('power_state')
             if state != 'Running' and state != 'Paused':
-                util.SMlog("vm " + vm_name + " is not running, cleaning up")
+                logging.debug("vm " + vm_name + " is not running, cleaning up")
                 destroy_network_rules_for_vm(session, {'vmName':vm_name})
                 cleaned = cleaned+1
                 
-    util.SMlog("Cleaned up rules for " + str(cleaned) + " vms")
+    logging.debug("Cleaned up rules for " + str(cleaned) + " vms")
   except:
-    util.SMlog("Failed to cleanup rules for dead vms!")
+    logging.debug("Failed to cleanup rules for dead vms!")
         
 
 @echo
@@ -1234,12 +1238,12 @@ def cleanup_rules(session, args):
     hostrec = session.xenapi.host.get_record(thishost[0])
     vms = hostrec.get('resident_VMs')
     resident_vms = [session.xenapi.VM.get_name_label(x) for x in vms]
-    util.SMlog('cleanup_rules: found %s resident vms on this host %s' % (len(resident_vms)-1, hostname[0]))
+    logging.debug('cleanup_rules: found %s resident vms on this host %s' % (len(resident_vms)-1, hostname[0]))
  
     chainscmd = "iptables-save | grep '^:' | awk '{print $1}' | cut -d':' -f2 | sed 's/-def/-%s/'| sed 's/-eg//' | sort|uniq" % instance
     chains = util.pread2(['/bin/bash', '-c', chainscmd]).split('\n')
     vmchains = [ch  for ch in chains if 1 in [ ch.startswith(c) for c in ['r-', 'i-', 's-', 'v-', 'l-']]]
-    util.SMlog('cleanup_rules: found %s iptables chains for vms on this host %s' % (len(vmchains), hostname[0]))
+    logging.debug('cleanup_rules: found %s iptables chains for vms on this host %s' % (len(vmchains), hostname[0]))
     cleaned = 0
     cleanup = []
     for chain in vmchains:
@@ -1247,16 +1251,16 @@ def cleanup_rules(session, args):
         if vmname not in resident_vms:
             vmname = chain + "-untagged"
             if vmname not in resident_vms:
-                util.SMlog("vm " + chain + " is not running on this host, cleaning up")
+                logging.debug("vm " + chain + " is not running on this host, cleaning up")
                 cleanup.append(chain)
 
     for vm_name in cleanup:
         destroy_network_rules_for_vm(session, {'vmName':vm_name})
                     
-    util.SMlog("Cleaned up rules for " + str(len(cleanup)) + " chains")
+    logging.debug("Cleaned up rules for " + str(len(cleanup)) + " chains")
     return str(len(cleanup))                
   except Exception, ex:
-    util.SMlog("Failed to cleanup rules, reason= " + str(ex))
+    logging.debug("Failed to cleanup rules, reason= " + str(ex))
     return '-1';
 
 @echo
@@ -1264,7 +1268,7 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno):
     vm_name = vmName;
     logfilename = "/var/run/cloud/" + vm_name +".log"
     if not os.path.exists(logfilename):
-        util.SMlog("Failed to find logfile %s" %logfilename)
+        logging.debug("Failed to find logfile %s" %logfilename)
         return [True, True, True]
         
     lines = (line.rstrip() for line in open(logfilename))
@@ -1278,39 +1282,39 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno):
                 [_vmName,_vmID,_vmIP,_domID,_signature,_seqno] = line.split(',')
             break
     except:
-        util.SMlog("Failed to parse log file for vm " + vmName)
+        logging.debug("Failed to parse log file for vm " + vmName)
         remove_rule_log_for_vm(vmName)
         return [True, True, True]
     
     reprogramDefault = False
     if (domID != _domID) or (vmID != _vmID) or (vmIP != _vmIP):
-        util.SMlog("Change in default info set of vm %s" % vmName)
+        logging.debug("Change in default info set of vm %s" % vmName)
         return [True, True, True]
     else:
-        util.SMlog("No change in default info set of vm %s" % vmName)
+        logging.debug("No change in default info set of vm %s" % vmName)
     
     reprogramChain = False
     rewriteLog = True
     if (int(seqno) > int(_seqno)):
         if (_signature != signature):
             reprogramChain = True
-            util.SMlog("Seqno increased from %s to %s: reprogamming "\
+            logging.debug("Seqno increased from %s to %s: reprogamming "\
                         "ingress rules for vm %s" % (_seqno, seqno, vmName))
         else:
-            util.SMlog("Seqno increased from %s to %s: but no change "\
+            logging.debug("Seqno increased from %s to %s: but no change "\
                         "in signature for vm: skip programming ingress "\
                         "rules %s" % (_seqno, seqno, vmName))
     elif (int(seqno) < int(_seqno)):
-        util.SMlog("Seqno decreased from %s to %s: ignoring these "\
+        logging.debug("Seqno decreased from %s to %s: ignoring these "\
                         "ingress rules for vm %s" % (_seqno, seqno, vmName))
         rewriteLog = False
     elif (signature != _signature):
-        util.SMlog("Seqno %s stayed the same but signature changed from "\
+        logging.debug("Seqno %s stayed the same but signature changed from "\
                     "%s to %s for vm %s" % (seqno, _signature, signature, vmName))
         rewriteLog = True
         reprogramChain = True
     else:
-        util.SMlog("Seqno and signature stayed the same: %s : ignoring these "\
+        logging.debug("Seqno and signature stayed the same: %s : ignoring these "\
                         "ingress rules for vm %s" % (seqno, vmName))
         rewriteLog = False
         
@@ -1320,7 +1324,7 @@ def check_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno):
 def write_secip_log_for_vm (vmName, secIps, vmId):
     vm_name = vmName
     logfilename = "/var/run/cloud/"+vm_name+".ip"
-    util.SMlog("Writing log to " + logfilename)
+    logging.debug("Writing log to " + logfilename)
     logf = open(logfilename, 'w')
     output = ','.join([vmName, secIps, vmId])
     result = True
@@ -1329,7 +1333,7 @@ def write_secip_log_for_vm (vmName, secIps, vmId):
         logf.write(output)
         logf.write('\n')
     except:
-        util.SMlog("Failed to write to rule log file " + logfilename)
+        logging.debug("Failed to write to rule log file " + logfilename)
         result = False
 
     logf.close()
@@ -1345,7 +1349,7 @@ def remove_secip_log_for_vm(vmName):
     try:
         os.remove(logfilename)
     except:
-        util.SMlog("Failed to delete rule log file " + logfilename)
+        logging.debug("Failed to delete rule log file " + logfilename)
         result = False
 
     return result
@@ -1354,7 +1358,7 @@ def remove_secip_log_for_vm(vmName):
 def write_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno, vmMac='ff:ff:ff:ff:ff:ff'):
     vm_name = vmName
     logfilename = "/var/run/cloud/" + vm_name +".log"
-    util.SMlog("Writing log to " + logfilename)
+    logging.debug("Writing log to " + logfilename)
     logf = open(logfilename, 'w')
     output = ','.join([vmName, vmID, vmIP, domID, signature, seqno, vmMac])
     result = True
@@ -1362,7 +1366,7 @@ def write_rule_log_for_vm(vmName, vmID, vmIP, domID, signature, seqno, vmMac='ff
         logf.write(output)
         logf.write('\n')
     except:
-        util.SMlog("Failed to write to rule log file " + logfilename)
+        logging.debug("Failed to write to rule log file " + logfilename)
         result = False
         
     logf.close()
@@ -1378,7 +1382,7 @@ def remove_rule_log_for_vm(vmName):
     try:
         os.remove(logfilename)
     except:
-        util.SMlog("Failed to delete rule log file " + logfilename)
+        logging.debug("Failed to delete rule log file " + logfilename)
         result = False
     
     return result
@@ -1408,13 +1412,13 @@ def cache_ipset_keyword():
        pass
        
     cachefile = "/var/cache/cloud/ipset.keyword"
-    util.SMlog("Writing ipset keyword to " + cachefile)
+    logging.debug("Writing ipset keyword to " + cachefile)
     cachef = open(cachefile, 'w')
     try:
         cachef.write(keyword)
         cachef.write('\n')
     except:
-        util.SMlog("Failed to write to cache file " + cachef)
+        logging.debug("Failed to write to cache file " + cachef)
         
     cachef.close()
     return keyword
@@ -1425,7 +1429,7 @@ def get_ipset_keyword():
     keyword = 'match-set'
     
     if not os.path.exists(cachefile):
-        util.SMlog("Failed to find ipset keyword cachefile %s" %cachefile)
+        logging.debug("Failed to find ipset keyword cachefile %s" %cachefile)
         keyword = cache_ipset_keyword()
     else:
         lines = (line.rstrip() for line in open(cachefile))
@@ -1458,15 +1462,15 @@ def network_rules(session, args):
     try:
         vm = session.xenapi.VM.get_by_name_label(vm_name)
         if len(vm) != 1:
-             util.SMlog("### Could not get record for vm ## " + vm_name)
+             logging.debug("### Could not get record for vm ## " + vm_name)
              return 'false'
         vm_rec = session.xenapi.VM.get_record(vm[0])
         domid = vm_rec.get('domid')
     except:
-        util.SMlog("### Failed to get domid for vm  ## " + vm_name)
+        logging.debug("### Failed to get domid for vm  ## " + vm_name)
         return 'false'
     if domid == '-1':
-        util.SMlog("### Failed to get domid for vm (-1):  " + vm_name)
+        logging.debug("### Failed to get domid for vm (-1):  " + vm_name)
         return 'false'
    
     vif = "vif" + domid + ".0"
@@ -1484,21 +1488,21 @@ def network_rules(session, args):
              check_rule_log_for_vm (vm_name, vm_id, vm_ip, domid, signature, seqno)
     
     if not reprogramDefault and not reprogramChain:
-        util.SMlog("No changes detected between current state and received state")
+        logging.debug("No changes detected between current state and received state")
         reason = 'seqno_same_sig_same'
         if rewriteLog:
             reason = 'seqno_increased_sig_same'
             write_rule_log_for_vm(vm_name, vm_id, vm_ip, domid, signature, seqno, vm_mac)
-        util.SMlog("Programming network rules for vm  %s seqno=%s signature=%s guestIp=%s,"\
+        logging.debug("Programming network rules for vm  %s seqno=%s signature=%s guestIp=%s,"\
                " do nothing, reason=%s" % (vm_name, seqno, signature, vm_ip, reason))
         return 'true'
            
     if not reprogramChain:
-        util.SMlog("###Not programming any ingress rules since no changes detected?")
+        logging.debug("###Not programming any ingress rules since no changes detected?")
         return 'true'
 
     if reprogramDefault:
-        util.SMlog("Change detected in vmId or vmIp or domId, resetting default rules")
+        logging.debug("Change detected in vmId or vmIp or domId, resetting default rules")
         default_network_rules(session, args)
         reason = 'domid_change'
     
@@ -1508,7 +1512,7 @@ def network_rules(session, args):
     keyword = '--' + get_ipset_keyword() 
     lines = rules.split(' ')
 
-    util.SMlog("Programming network rules for vm  %s seqno=%s numrules=%s signature=%s guestIp=%s,"\
+    logging.debug("Programming network rules for vm  %s seqno=%s numrules=%s signature=%s guestIp=%s,"\
               " update iptables, reason=%s" % (vm_name, seqno, len(lines), signature, vm_ip, reason))
     
     cmds = []
@@ -1546,7 +1550,7 @@ def network_rules(session, args):
                 ipsetname = vmchain + "_" + protocol + "_any"
 
             if ipset(ipsetname, protocol, start, end, ips) == False:
-                util.SMlog(" failed to create ipset for rule " + str(tokens))
+                logging.debug(" failed to create ipset for rule " + str(tokens))
 
             if protocol == 'all':
                 iptables = ['iptables', '-I', vmchain, '-m', 'state', '--state', 'NEW', '-m', 'set', keyword, ipsetname, direction, '-j', action]
@@ -1559,7 +1563,7 @@ def network_rules(session, args):
                 iptables = ['iptables', '-I', vmchain, '-p',  'icmp', '--icmp-type',  range,  '-m', 'set', keyword, ipsetname, direction, '-j', action]
                 
             cmds.append(iptables)
-            util.SMlog(iptables)
+            logging.debug(iptables)
         
         if allow_any and protocol != 'all':
             if protocol != 'icmp':
@@ -1570,20 +1574,20 @@ def network_rules(session, args):
                     range = "any"
                 iptables = ['iptables', '-I', vmchain, '-p',  'icmp', '--icmp-type',  range, '-j', action]
             cmds.append(iptables)
-            util.SMlog(iptables)
+            logging.debug(iptables)
       
     vmchain = chain_name(vm_name)
     try:
         util.pread2(['iptables', '-F', vmchain])
     except:
-        util.SMlog("Ignoring failure to delete chain " + vmchain)
+        logging.debug("Ignoring failure to delete chain " + vmchain)
         util.pread2(['iptables', '-N', vmchain])
 
     egress_vmchain = egress_chain_name(vm_name)
     try:
         util.pread2(['iptables', '-F', egress_vmchain])
     except:
-        util.SMlog("Ignoring failure to delete chain " + egress_vmchain)
+        logging.debug("Ignoring failure to delete chain " + egress_vmchain)
         util.pread2(['iptables', '-N', egress_vmchain])
 
     
@@ -1602,7 +1606,7 @@ def network_rules(session, args):
     
     return 'true'
   except:
-    util.SMlog("Failed to network rule !")
+    logging.debug("Failed to network rule !")
 
 @echo
 def bumpUpPriority(session, args):
@@ -1614,7 +1618,7 @@ def bumpUpPriority(session, args):
         txt = util.pread2(cmd)
         txt = 'success'
     except:
-        util.SMlog("bump up priority fail! ")
+        logging.debug("bump up priority fail! ")
         txt = ''
 
     return txt

scripts/vm/hypervisor/xenserver/vmopsSnapshot

@@ -38,6 +38,10 @@ import xs_errors
 import cleanup
 import stat
 import random
+import cloudstack_pluginlib as lib
+import logging
+
+lib.setup_logging("/var/log/cloud/vmops.log")
 
 VHDUTIL = "vhd-util"
 VHD_PREFIX = 'VHD-'
@@ -46,9 +50,9 @@ CLOUD_DIR = '/var/run/cloud_mount'
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### VMOPS enter  %s ####" % name )
+        logging.debug("#### VMOPS enter  %s ####" % name )
         res = fn(*v, **k)
-        util.SMlog("#### VMOPS exit  %s ####" % name )
+        logging.debug("#### VMOPS exit  %s ####" % name )
         return res
     return wrapped
 
@@ -57,7 +61,7 @@ def echo(fn):
 def create_secondary_storage_folder(session, args):
     local_mount_path = None
 
-    util.SMlog("create_secondary_storage_folder, args: " + str(args))
+    logging.debug("create_secondary_storage_folder, args: " + str(args))
 
     try:
         try:
@@ -74,11 +78,11 @@ def create_secondary_storage_folder(session, args):
                 os.umask(current_umask)
         except OSError, (errno, strerror):
             errMsg = "create_secondary_storage_folder failed: errno: " + str(errno) + ", strerr: " + strerror
-            util.SMlog(errMsg)
+            logging.debug(errMsg)
             raise xs_errors.XenError(errMsg)
         except:
             errMsg = "create_secondary_storage_folder failed."
-            util.SMlog(errMsg)
+            logging.debug(errMsg)
             raise xs_errors.XenError(errMsg)
     finally:
         if local_mount_path != None:
@@ -93,7 +97,7 @@ def create_secondary_storage_folder(session, args):
 def delete_secondary_storage_folder(session, args):
     local_mount_path = None
 
-    util.SMlog("delete_secondary_storage_folder, args: " + str(args))
+    logging.debug("delete_secondary_storage_folder, args: " + str(args))
 
     try:
         try:
@@ -109,11 +113,11 @@ def delete_secondary_storage_folder(session, args):
                 os.system("rmdir " + folder)
         except OSError, (errno, strerror):
             errMsg = "delete_secondary_storage_folder failed: errno: " + str(errno) + ", strerr: " + strerror
-            util.SMlog(errMsg)
+            logging.debug(errMsg)
             raise xs_errors.XenError(errMsg)
         except:
             errMsg = "delete_secondary_storage_folder failed."
-            util.SMlog(errMsg)
+            logging.debug(errMsg)
             raise xs_errors.XenError(errMsg)
     finally:
         if local_mount_path != None:
@@ -159,16 +163,16 @@ def post_create_private_template(session, args):
             f.write("vhd.size=" + str(file_size) + "\n")
             f.write("size=" + str(file_size) + "\n")
             f.close()
-            util.SMlog("Created template.properties file")
+            logging.debug("Created template.properties file")
            
             # Set permissions
             permissions = stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH
             os.chmod(template_properties_install_path, permissions)
-            util.SMlog("Set permissions on template and template.properties")
+            logging.debug("Set permissions on template and template.properties")
 
         except:
             errMsg = "post_create_private_template failed."
-            util.SMlog(errMsg)
+            logging.debug(errMsg)
             raise xs_errors.XenError(errMsg)
 
     finally:
@@ -189,12 +193,12 @@ def isfile(path, isISCSI):
         
     if not exists:
         errMsg = "File " + path + " does not exist."
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
     return errMsg
 
 def copyfile(fromFile, toFile, isISCSI):
-    util.SMlog("Starting to copy " + fromFile + " to " + toFile)
+    logging.debug("Starting to copy " + fromFile + " to " + toFile)
     errMsg = ''
     try:
         cmd = ['dd', 'if=' + fromFile, 'of=' + toFile, 'bs=4M']
@@ -206,10 +210,10 @@ def copyfile(fromFile, toFile, isISCSI):
             txt = ''
         txt = ''
         errMsg = "Error while copying " + fromFile + " to " + toFile + " in secondary storage"
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
 
-    util.SMlog("Successfully copied " + fromFile + " to " + toFile)
+    logging.debug("Successfully copied " + fromFile + " to " + toFile)
     return errMsg
 
 def chdir(path):
@@ -217,9 +221,9 @@ def chdir(path):
         os.chdir(path)
     except OSError, (errno, strerror):
         errMsg = "Unable to chdir to " + path + " because of OSError with errno: " + str(errno) + " and strerr: " + strerror
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
-    util.SMlog("Chdired to " + path)
+    logging.debug("Chdired to " + path)
     return
 
 def scanParent(path):
@@ -234,7 +238,7 @@ def scanParent(path):
         parentUUID = vhdInfo.parentUuid
     except:
         errMsg = "Could not get vhd parent of " + path
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
     return parentUUID
 
@@ -247,7 +251,7 @@ def getParent(path, isISCSI):
             parentUUID = vhdutil.getParent(path, cleanup.FileVDI.extractUuid)
     except:
         errMsg = "Could not get vhd parent of " + path
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
     return parentUUID
 
@@ -262,7 +266,7 @@ def getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI):
     else:
         baseCopyUuid = getParent(snapshotPath, isISCSI)
     
-    util.SMlog("Base copy of snapshotUuid: " + snapshotUuid + " is " + baseCopyUuid)
+    logging.debug("Base copy of snapshotUuid: " + snapshotUuid + " is " + baseCopyUuid)
     return baseCopyUuid
 
 def setParent(parent, child):
@@ -271,9 +275,9 @@ def setParent(parent, child):
         txt = util.pread2(cmd)
     except:
         errMsg = "Unexpected error while trying to set parent of " + child + " to " + parent 
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
-    util.SMlog("Successfully set parent of " + child + " to " + parent)
+    logging.debug("Successfully set parent of " + child + " to " + parent)
     return
 
 def rename(originalVHD, newVHD):
@@ -281,7 +285,7 @@ def rename(originalVHD, newVHD):
         os.rename(originalVHD, newVHD)
     except OSError, (errno, strerror):
         errMsg = "OSError while renaming " + origiinalVHD + " to " + newVHD + "with errno: " + str(errno) + " and strerr: " + strerror
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
     return
 
@@ -294,7 +298,7 @@ def makedirs(path):
             if os.path.isdir(path):
                 return
             errMsg = "OSError while creating " + path + " with errno: " + str(errno) + " and strerr: " + strerror
-            util.SMlog(errMsg)
+            logging.debug(errMsg)
             raise xs_errors.XenError(errMsg)
     return
 
@@ -307,9 +311,9 @@ def mount(remoteDir, localDir):
     except:
         txt = ''
         errMsg = "Unexpected error while trying to mount " + remoteDir + " to " + localDir 
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
-    util.SMlog("Successfully mounted " + remoteDir + " to " + localDir)
+    logging.debug("Successfully mounted " + remoteDir + " to " + localDir)
 
     return
 
@@ -319,10 +323,10 @@ def umount(localDir):
         util.pread2(cmd)
     except CommandException:
         errMsg = "CommandException raised while trying to umount " + localDir 
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
 
-    util.SMlog("Successfully unmounted " + localDir)
+    logging.debug("Successfully unmounted " + localDir)
     return
 
 def mountSnapshotsDir(secondaryStorageMountPath, localMountPointPath, path):
@@ -344,7 +348,7 @@ def mountSnapshotsDir(secondaryStorageMountPath, localMountPointPath, path):
         # There is more than one secondary storage per zone.
         # And we are mounting each sec storage under a zone-specific directory
         # So two secondary storage snapshot dirs will never get mounted on the same point on the same XenServer.
-        util.SMlog("The remote snapshots directory has already been mounted on " + localMountPointPath)
+        logging.debug("The remote snapshots directory has already been mounted on " + localMountPointPath)
     else:
         mount(snapshotsDir, localMountPointPath)
 
@@ -357,11 +361,11 @@ def unmountAll(path):
     try:
         for dir in os.listdir(path):
             if dir.isdigit():
-                util.SMlog("Unmounting Sub-Directory: " + dir)
+                logging.debug("Unmounting Sub-Directory: " + dir)
                 localMountPointPath = os.path.join(path, dir)
                 umount(localMountPointPath)
     except:
-        util.SMlog("Ignoring the error while trying to unmount the snapshots dir")
+        logging.debug("Ignoring the error while trying to unmount the snapshots dir")
 
 @echo
 def unmountSnapshotsDir(session, args):
@@ -372,7 +376,7 @@ def unmountSnapshotsDir(session, args):
     try:
         umount(localMountPointPath)
     except:
-        util.SMlog("Ignoring the error while trying to unmount the snapshots dir.")
+        logging.debug("Ignoring the error while trying to unmount the snapshots dir.")
 
     return "1"
 
@@ -409,14 +413,14 @@ def makeUnavailable(uuid, primarySRPath, isISCSI):
 def manageAvailability(path, value):
     if path.__contains__("/var/run/sr-mount"):
         return
-    util.SMlog("Setting availability of " + path + " to " + value)
+    logging.debug("Setting availability of " + path + " to " + value)
     try:
         cmd = ['/usr/sbin/lvchange', value, path]
         util.pread2(cmd)
     except: #CommandException, (rc, cmdListStr, stderr):
         #errMsg = "CommandException thrown while executing: " + cmdListStr + " with return code: " + str(rc) + " and stderr: " + stderr
         errMsg = "Unexpected exception thrown by lvchange"
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         if value == "-ay":
             # Raise an error only if we are trying to make it available.
             # Just warn if we are trying to make it unavailable after the 
@@ -434,7 +438,7 @@ def checkVolumeAvailablility(path):
             manageAvailability(path, '-ay')
     except:
         errMsg = "Could not determine status of ISCSI path: " + path
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
     
     success = False
@@ -445,7 +449,7 @@ def checkVolumeAvailablility(path):
         # set isISCSI to true
         success = isVolumeAvailable(path)
         if success:
-            util.SMlog("Made vhd: " + path + " available and confirmed that it is visible")
+            logging.debug("Made vhd: " + path + " available and confirmed that it is visible")
             break
 
         # Sleep for 10 seconds before checking again.
@@ -453,7 +457,7 @@ def checkVolumeAvailablility(path):
 
     # If not visible within 1 min fail
     if not success:
-        util.SMlog("Could not make vhd: " +  path + " available despite waiting for 1 minute. Does it exist?")
+        logging.debug("Could not make vhd: " +  path + " available despite waiting for 1 minute. Does it exist?")
 
     return success
 
@@ -465,19 +469,19 @@ def isVolumeAvailable(path):
         status = p.communicate()[0].strip("\n")
     except:
         errMsg = "Could not determine status of ISCSI path: " + path
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
 
     return (status == "1")  
 
 def getVhdParent(session, args):
-    util.SMlog("getParent with " + str(args))
+    logging.debug("getParent with " + str(args))
     primaryStorageSRUuid      = args['primaryStorageSRUuid']
     snapshotUuid              = args['snapshotUuid']
     isISCSI                   = getIsTrueString(args['isISCSI']) 
 
     primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
-    util.SMlog("primarySRPath: " + primarySRPath)
+    logging.debug("primarySRPath: " + primarySRPath)
 
     baseCopyUuid = getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI)
 
@@ -485,7 +489,7 @@ def getVhdParent(session, args):
 
 
 def backupSnapshot(session, args):
-    util.SMlog("Called backupSnapshot with " + str(args))
+    logging.debug("Called backupSnapshot with " + str(args))
     primaryStorageSRUuid      = args['primaryStorageSRUuid']
     secondaryStorageMountPath = args['secondaryStorageMountPath']
     snapshotUuid              = args['snapshotUuid']
@@ -495,19 +499,19 @@ def backupSnapshot(session, args):
     path = args['path']
     localMountPoint = args['localMountPoint']
     primarySRPath = getPrimarySRPath(primaryStorageSRUuid, isISCSI)
-    util.SMlog("primarySRPath: " + primarySRPath)
+    logging.debug("primarySRPath: " + primarySRPath)
 
     baseCopyUuid = getParentOfSnapshot(snapshotUuid, primarySRPath, isISCSI)
     baseCopyVHD  = getVHD(baseCopyUuid, isISCSI)
     baseCopyPath = os.path.join(primarySRPath, baseCopyVHD)
-    util.SMlog("Base copy path: " + baseCopyPath)
+    logging.debug("Base copy path: " + baseCopyPath)
 
 
     # Mount secondary storage mount path on XenServer along the path
     # /var/run/sr-mount/<dcId>/snapshots/ and create <accountId>/<volumeId> dir
     # on it.
     backupsDir = mountSnapshotsDir(secondaryStorageMountPath, localMountPoint, path)
-    util.SMlog("Backups dir " + backupsDir)
+    logging.debug("Backups dir " + backupsDir)
     prevBackupUuid = prevBackupUuid.split("/")[-1]
     # Check existence of snapshot on primary storage
     isfile(baseCopyPath, isISCSI)
@@ -520,7 +524,7 @@ def backupSnapshot(session, args):
     # copy baseCopyPath to backupsDir with new uuid
     backupVHD = getBackupVHD(backupUuid)  
     backupFile = os.path.join(backupsDir, backupVHD)
-    util.SMlog("Back up " + baseCopyUuid + " to Secondary Storage as " + backupUuid)
+    logging.debug("Back up " + baseCopyUuid + " to Secondary Storage as " + backupUuid)
     copyfile(baseCopyPath, backupFile, isISCSI)
     vhdutil.setHidden(backupFile, False)
 
@@ -536,7 +540,7 @@ def backupSnapshot(session, args):
 
 @echo
 def deleteSnapshotBackup(session, args):
-    util.SMlog("Calling deleteSnapshotBackup with " + str(args))
+    logging.debug("Calling deleteSnapshotBackup with " + str(args))
     secondaryStorageMountPath = args['secondaryStorageMountPath']
     backupUUID                = args['backupUUID']
     path = args['path']
@@ -547,27 +551,27 @@ def deleteSnapshotBackup(session, args):
     chdir(backupsDir)
 
     backupVHD = getBackupVHD(backupUUID)
-    util.SMlog("checking existence of " + backupVHD)
+    logging.debug("checking existence of " + backupVHD)
 
     # The backupVHD is on secondary which is NFS and not ISCSI.
     if not os.path.isfile(backupVHD):
-        util.SMlog("backupVHD " + backupVHD + "does not exist. Not trying to delete it")
+        logging.debug("backupVHD " + backupVHD + "does not exist. Not trying to delete it")
         return "1"
-    util.SMlog("backupVHD " + backupVHD + " exists.")
+    logging.debug("backupVHD " + backupVHD + " exists.")
         
     # Just delete the backupVHD
     try:
         os.remove(backupVHD)
     except OSError, (errno, strerror):
         errMsg = "OSError while removing " + backupVHD + " with errno: " + str(errno) + " and strerr: " + strerror
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
 
     return "1"
    
 @echo
 def revert_memory_snapshot(session, args):
-    util.SMlog("Calling revert_memory_snapshot with " + str(args))
+    logging.debug("Calling revert_memory_snapshot with " + str(args))
     vmName = args['vmName']
     snapshotUUID = args['snapshotUUID']
     oldVmUuid = args['oldVmUuid']
@@ -587,7 +591,7 @@ def revert_memory_snapshot(session, args):
             os.system("xe vdi-destroy uuid=" + vdiUuid)
     except OSError, (errno, strerror):
         errMsg = "OSError while reverting vm " + vmName + " to snapshot " + snapshotUUID + " with errno: " + str(errno) + " and strerr: " + strerror
-        util.SMlog(errMsg)
+        logging.debug(errMsg)
         raise xs_errors.XenError(errMsg)
     return "0"
 

scripts/vm/hypervisor/xenserver/vmopspremium

@@ -28,13 +28,18 @@ if os.path.exists("/usr/lib/xcp/sm"):
     sys.path.extend(["/usr/lib/xcp/sm/", "/usr/local/sbin/", "/sbin/"])
 import util
 import socket
+import perfmon
+import cloudstack_pluginlib as lib
+import logging
+
+lib.setup_logging("/var/log/cloud/vmops.log")
 
 def echo(fn):
     def wrapped(*v, **k):
         name = fn.__name__
-        util.SMlog("#### VMOPS enter  %s ####" % name )
+        logging.debug("#### VMOPS enter  %s ####" % name )
         res = fn(*v, **k)
-        util.SMlog("#### VMOPS exit  %s ####" % name )
+        logging.debug("#### VMOPS exit  %s ####" % name )
         return res
     return wrapped
 
@@ -142,6 +147,11 @@ def heartbeat(session, args):
        txt='fail'
     return txt
 
-if __name__ == "__main__":
-    XenAPIPlugin.dispatch({"forceShutdownVM":forceShutdownVM, "upgrade_snapshot":upgrade_snapshot, "create_privatetemplate_from_snapshot":create_privatetemplate_from_snapshot, "copy_vhd_to_secondarystorage":copy_vhd_to_secondarystorage, "copy_vhd_from_secondarystorage":copy_vhd_from_secondarystorage, "setup_heartbeat_sr":setup_heartbeat_sr, "setup_heartbeat_file":setup_heartbeat_file, "check_heartbeat":check_heartbeat, "heartbeat": heartbeat})
+@echo
+def asmonitor(session, args):
+    result = perfmon.get_vm_group_perfmon(args)
+    return result
+
+if __name__ == "__main__":
+    XenAPIPlugin.dispatch({"forceShutdownVM":forceShutdownVM, "upgrade_snapshot":upgrade_snapshot, "create_privatetemplate_from_snapshot":create_privatetemplate_from_snapshot, "copy_vhd_to_secondarystorage":copy_vhd_to_secondarystorage, "copy_vhd_from_secondarystorage":copy_vhd_from_secondarystorage, "setup_heartbeat_sr":setup_heartbeat_sr, "setup_heartbeat_file":setup_heartbeat_file, "check_heartbeat":check_heartbeat, "heartbeat": heartbeat, "asmonitor": asmonitor})
 

scripts/vm/hypervisor/xenserver/xcposs/patch

@@ -70,3 +70,4 @@ deleteipAlias.sh=..,0755,/opt/cloud/bin
 ###add cloudstack plugin script for XCP
 cloudstack_plugins.conf=..,0644,/etc/xensource
 cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins
+cloudlog=..,0644,/etc/logrotate.d

scripts/vm/hypervisor/xenserver/xcpserver/patch

@@ -68,3 +68,4 @@ add_to_vcpus_params_live.sh=..,0755,/opt/cloud/bin
 ###add cloudstack plugin script for XCP
 cloudstack_plugins.conf=..,0644,/etc/xensource
 cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins
+cloudlog=..,0644,/etc/logrotate.d

scripts/vm/hypervisor/xenserver/xenserver56/patch

@@ -69,3 +69,4 @@ add_to_vcpus_params_live.sh=..,0755,/opt/cloud/bin
 ###add cloudstack plugin script for XCP
 cloudstack_plugins.conf=..,0644,/etc/xensource
 cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins
+cloudlog=..,0644,/etc/logrotate.d

scripts/vm/hypervisor/xenserver/xenserver56fp1/patch

@@ -68,4 +68,4 @@ add_to_vcpus_params_live.sh=..,0755,/opt/cloud/bin
 ###add cloudstack plugin script for XCP
 cloudstack_plugins.conf=..,0644,/etc/xensource
 cloudstack_pluginlib.py=..,0755,/etc/xapi.d/plugins
-
+cloudlog=..,0644,/etc/logrotate.d

scripts/vm/hypervisor/xenserver/xenserver60/patch

@@ -75,3 +75,4 @@ ovs-pvlan-vm.sh=../../../network,0755,/opt/cloud/bin
 ovs-pvlan-cleanup.sh=../../../network,0755,/opt/cloud/bin
 ovs-get-dhcp-iface.sh=..,0755,/opt/cloud/bin
 ovs-get-bridge.sh=..,0755,/opt/cloud/bin
+cloudlog=..,0644,/etc/logrotate.d

server/src/com/cloud/capacity/CapacityManagerImpl.java

@@ -27,7 +27,10 @@ import javax.ejb.Local;
 import javax.inject.Inject;
 import javax.naming.ConfigurationException;
 
+import com.cloud.deploy.DeploymentClusterPlanner;
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.event.UsageEventVO;
+import com.cloud.utils.exception.CloudRuntimeException;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.framework.config.ConfigDepot;
@@ -94,6 +97,7 @@ import com.cloud.vm.dao.UserVmDao;
 import com.cloud.vm.dao.UserVmDetailsDao;
 import com.cloud.vm.dao.VMInstanceDao;
 import com.cloud.vm.snapshot.dao.VMSnapshotDao;
+import org.springframework.instrument.classloading.glassfish.GlassFishLoadTimeWeaver;
 
 @Local(value = CapacityManager.class)
 public class CapacityManagerImpl extends ManagerBase implements CapacityManager, StateListener<State, VirtualMachine.Event, VirtualMachine>, Listener, ResourceListener,
@@ -852,6 +856,50 @@ public class CapacityManagerImpl extends ManagerBase implements CapacityManager,
 
     }
 
+    @Override
+    public float getClusterOverProvisioningFactor(Long clusterId, short capacityType){
+
+        String capacityOverProvisioningName = "";
+        if(capacityType == Capacity.CAPACITY_TYPE_CPU){
+            capacityOverProvisioningName = "cpuOvercommitRatio";
+        }else if(capacityType == Capacity.CAPACITY_TYPE_MEMORY){
+            capacityOverProvisioningName = "memoryOvercommitRatio";
+        }else{
+            throw new CloudRuntimeException("Invalid capacityType - " + capacityType);
+        }
+
+        ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, capacityOverProvisioningName);
+        Float clusterOverProvisioningRatio = Float.parseFloat(clusterDetailCpu.getValue());
+        return clusterOverProvisioningRatio;
+
+    }
+
+    @Override
+    public boolean checkIfClusterCrossesThreshold(Long clusterId, Integer cpuRequested, long ramRequested){
+
+        Float clusterCpuOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_CPU);
+        Float clusterMemoryOverProvisioning = getClusterOverProvisioningFactor(clusterId, Capacity.CAPACITY_TYPE_MEMORY);
+        Float clusterCpuCapacityDisableThreshold = DeploymentClusterPlanner.ClusterCPUCapacityDisableThreshold.valueIn(clusterId);
+        Float clusterMemoryCapacityDisableThreshold = DeploymentClusterPlanner.ClusterMemoryCapacityDisableThreshold.valueIn(clusterId);
+
+        float cpuConsumption = _capacityDao.findClusterConsumption(clusterId, Capacity.CAPACITY_TYPE_CPU, cpuRequested);
+        if(cpuConsumption/clusterCpuOverProvisioning > clusterCpuCapacityDisableThreshold){
+            s_logger.debug("Cluster: " +clusterId + " cpu consumption " + cpuConsumption/clusterCpuOverProvisioning
+                    + " crosses disable threshold " + clusterCpuCapacityDisableThreshold);
+            return true;
+        }
+
+        float memoryConsumption = _capacityDao.findClusterConsumption(clusterId, Capacity.CAPACITY_TYPE_MEMORY, ramRequested);
+        if(memoryConsumption/clusterMemoryOverProvisioning > clusterMemoryCapacityDisableThreshold){
+            s_logger.debug("Cluster: " +clusterId + " memory consumption " + memoryConsumption/clusterMemoryOverProvisioning
+                    + " crosses disable threshold " + clusterMemoryCapacityDisableThreshold);
+            return true;
+        }
+
+        return false;
+
+    }
+
     @Override
     public boolean processAnswers(long agentId, long seq, Answer[] answers) {
         // TODO Auto-generated method stub

server/src/com/cloud/configuration/Config.java

@@ -2009,6 +2009,8 @@ public enum Config {
             return "VpcManager";
         } else if (_componentClass == SnapshotManager.class) {
             return "SnapshotManager";
+        } else if (_componentClass == VMSnapshotManager.class) {
+            return "VMSnapshotManager";
         } else {
             return "none";
         }

server/src/com/cloud/configuration/ConfigurationManagerImpl.java

@@ -3718,9 +3718,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati
             Set<Provider> firewallProviderSet = new HashSet<Provider>();
             firewallProviderSet.add(firewallProvider);
             serviceProviderMap.put(Service.Firewall, firewallProviderSet);
-            if (!(firewallProvider.getName().equals(Provider.JuniperSRX.getName()) || firewallProvider.getName().equals(Provider.VirtualRouter.getName())) &&
+            if (!(firewallProvider.getName().equals(Provider.JuniperSRX.getName()) || firewallProvider.getName().equals(Provider.PaloAlto.getName()) || firewallProvider.getName().equals(Provider.VirtualRouter.getName())) &&
                 egressDefaultPolicy == false) {
-                throw new InvalidParameterValueException("Firewall egress with default policy " + egressDefaultPolicy + "is not supported by the provider " +
+                throw new InvalidParameterValueException("Firewall egress with default policy " + egressDefaultPolicy + " is not supported by the provider " +
                     firewallProvider.getName());
             }
         }

server/src/com/cloud/deploy/DeploymentPlanningManagerImpl.java

@@ -19,8 +19,10 @@ package com.cloud.deploy;
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Timer;
 import java.util.TreeSet;
 
@@ -1119,6 +1121,12 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy
 
         // for each volume find list of suitable storage pools by calling the
         // allocators
+        Set<Long> originalAvoidPoolSet = avoid.getPoolsToAvoid();
+        if (originalAvoidPoolSet == null) {
+            originalAvoidPoolSet = new HashSet<Long>();
+        }
+        Set<Long> poolsToAvoidOutput = new HashSet<Long>(originalAvoidPoolSet);
+
         for (VolumeVO toBeCreated : volumesTobeCreated) {
             s_logger.debug("Checking suitable pools for volume (Id, Type): (" + toBeCreated.getId() + "," + toBeCreated.getVolumeType().name() + ")");
 
@@ -1228,6 +1236,11 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy
                 }
             }
 
+            if (avoid.getPoolsToAvoid() != null) {
+                poolsToAvoidOutput.addAll(avoid.getPoolsToAvoid());
+                avoid.getPoolsToAvoid().retainAll(originalAvoidPoolSet);
+            }
+
             if (!foundPotentialPools) {
                 s_logger.debug("No suitable pools found for volume: " + toBeCreated + " under cluster: " + plan.getClusterId());
                 // No suitable storage pools found under this cluster for this
@@ -1239,6 +1252,13 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy
             }
         }
 
+        if (suitableVolumeStoragePools.values() != null) {
+            poolsToAvoidOutput.removeAll(suitableVolumeStoragePools.values());
+        }
+        if (avoid.getPoolsToAvoid() != null) {
+            avoid.getPoolsToAvoid().addAll(poolsToAvoidOutput);
+        }
+
         if (suitableVolumeStoragePools.isEmpty()) {
             s_logger.debug("No suitable pools found");
         }
@@ -1346,4 +1366,4 @@ public class DeploymentPlanningManagerImpl extends ManagerBase implements Deploy
         }
         return true;
     }
-}
+}

server/src/com/cloud/network/ExternalLoadBalancerDeviceManager.java

@@ -47,7 +47,7 @@ public interface ExternalLoadBalancerDeviceManager extends Manager {
      * @return Host object for the device added
      */
     public ExternalLoadBalancerDeviceVO addExternalLoadBalancer(long physicalNetworkId, String url, String username, String password, String deviceName,
-        ServerResource resource, boolean gslbProvider, String gslbSitePublicIp, String gslbSitePrivateIp);
+        ServerResource resource, boolean gslbProvider, boolean exclusiveGslbProvider, String gslbSitePublicIp, String gslbSitePrivateIp);
 
     /**
      * deletes load balancer device added in to a physical network

server/src/com/cloud/network/ExternalLoadBalancerDeviceManagerImpl.java

@@ -198,7 +198,8 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase
     @Override
     @DB
     public ExternalLoadBalancerDeviceVO addExternalLoadBalancer(long physicalNetworkId, String url, String username, String password, final String deviceName,
-        ServerResource resource, final boolean gslbProvider, final String gslbSitePublicIp, final String gslbSitePrivateIp) {
+        ServerResource resource, final boolean gslbProvider, final boolean exclusiveGslbProivider,
+        final String gslbSitePublicIp, final String gslbSitePrivateIp) {
 
         PhysicalNetworkVO pNetwork = null;
         final NetworkDevice ntwkDevice = NetworkDevice.getNetworkDevice(deviceName);
@@ -285,6 +286,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase
                         if (gslbProvider) {
                             lbDeviceVO.setGslbSitePublicIP(gslbSitePublicIp);
                             lbDeviceVO.setGslbSitePrivateIP(gslbSitePrivateIp);
+                            lbDeviceVO.setExclusiveGslbProvider(exclusiveGslbProivider);
                         }
                         _externalLoadBalancerDeviceDao.persist(lbDeviceVO);
                         DetailVO hostDetail = new DetailVO(host.getId(), ApiConstants.LOAD_BALANCER_DEVICE_ID, String.valueOf(lbDeviceVO.getId()));
@@ -528,7 +530,7 @@ public abstract class ExternalLoadBalancerDeviceManagerImpl extends AdapterBase
                             try {
                                 lbAppliance =
                                     addExternalLoadBalancer(physicalNetworkId, url, username, password, createLbAnswer.getDeviceName(),
-                                        createLbAnswer.getServerResource(), false, null, null);
+                                        createLbAnswer.getServerResource(), false, false, null, null);
                             } catch (Exception e) {
                                 s_logger.error("Failed to add load balancer appliance in to cloudstack due to " + e.getMessage() +
                                     ". So provisioned load balancer appliance will be destroyed.");

server/src/com/cloud/network/NetworkModelImpl.java

@@ -2183,4 +2183,17 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel {
 
         return true;
     }
+
+    @Override
+    public boolean getNetworkEgressDefaultPolicy (Long networkId) {
+        NetworkVO network = _networksDao.findById(networkId);
+
+        if (network != null) {
+            NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId());
+            return offering.getEgressDefaultPolicy();
+        } else {
+            InvalidParameterValueException ex = new InvalidParameterValueException("network with network id does not exist");
+            throw ex;
+        }
+    }
 }

server/src/com/cloud/network/as/AutoScaleManager.java

@@ -19,4 +19,8 @@ package com.cloud.network.as;
 public interface AutoScaleManager extends AutoScaleService {
 
     void cleanUpAutoScaleResources(Long accountId);
+
+	void doScaleUp(long groupId, Integer numVm);
+
+	void doScaleDown(long groupId);
 }

server/src/com/cloud/network/as/AutoScaleManagerImpl.java

@@ -17,22 +17,25 @@
 package com.cloud.network.as;
 
 import java.security.InvalidParameterException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 import javax.ejb.Local;
 import javax.inject.Inject;
 
-import org.apache.log4j.Logger;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.ApiErrorCode;
+import org.apache.cloudstack.api.BaseCmd.HTTPMethod;
 import org.apache.cloudstack.api.BaseListAccountResourcesCmd;
+import org.apache.cloudstack.api.ServerApiException;
 import org.apache.cloudstack.api.command.admin.autoscale.CreateCounterCmd;
 import org.apache.cloudstack.api.command.user.autoscale.CreateAutoScalePolicyCmd;
 import org.apache.cloudstack.api.command.user.autoscale.CreateAutoScaleVmGroupCmd;
@@ -49,42 +52,55 @@ import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScaleVmProfile
 import org.apache.cloudstack.api.command.user.vm.DeployVMCmd;
 import org.apache.cloudstack.context.CallContext;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.log4j.Logger;
 
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.ApiDispatcher;
 import com.cloud.configuration.Config;
 import com.cloud.configuration.ConfigurationManager;
 import com.cloud.dc.DataCenter;
+import com.cloud.dc.DataCenter.NetworkType;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.event.ActionEvent;
 import com.cloud.event.EventTypes;
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InsufficientServerCapacityException;
 import com.cloud.exception.InvalidParameterValueException;
+import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceInUseException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.network.Network.Capability;
+import com.cloud.network.Network.IpAddresses;
 import com.cloud.network.as.AutoScaleCounter.AutoScaleCounterParam;
 import com.cloud.network.as.dao.AutoScalePolicyConditionMapDao;
 import com.cloud.network.as.dao.AutoScalePolicyDao;
 import com.cloud.network.as.dao.AutoScaleVmGroupDao;
 import com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDao;
+import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao;
 import com.cloud.network.as.dao.AutoScaleVmProfileDao;
 import com.cloud.network.as.dao.ConditionDao;
 import com.cloud.network.as.dao.CounterDao;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.LoadBalancerDao;
 import com.cloud.network.dao.LoadBalancerVMMapDao;
+import com.cloud.network.dao.LoadBalancerVMMapVO;
 import com.cloud.network.dao.LoadBalancerVO;
 import com.cloud.network.dao.NetworkDao;
 import com.cloud.network.lb.LoadBalancingRulesManager;
+import com.cloud.network.lb.LoadBalancingRulesService;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.projects.Project.ListProjectResourcesCriteria;
 import com.cloud.template.TemplateManager;
 import com.cloud.template.VirtualMachineTemplate;
 import com.cloud.user.Account;
 import com.cloud.user.AccountManager;
+import com.cloud.user.AccountService;
 import com.cloud.user.User;
 import com.cloud.user.dao.AccountDao;
 import com.cloud.user.dao.UserDao;
+import com.cloud.uservm.UserVm;
 import com.cloud.utils.Pair;
 import com.cloud.utils.Ternary;
 import com.cloud.utils.component.ManagerBase;
@@ -100,10 +116,15 @@ import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallback;
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.net.NetUtils;
+import com.cloud.vm.UserVmManager;
+import com.cloud.vm.UserVmService;
+import com.google.gson.Gson;
+import com.google.gson.reflect.TypeToken;
 
 @Local(value = {AutoScaleService.class, AutoScaleManager.class})
 public class AutoScaleManagerImpl<Type> extends ManagerBase implements AutoScaleManager, AutoScaleService {
     private static final Logger s_logger = Logger.getLogger(AutoScaleManagerImpl.class);
+    private ScheduledExecutorService _executor = Executors.newScheduledThreadPool(1);
 
     @Inject
     EntityManager _entityMgr;
@@ -138,6 +159,8 @@ public class AutoScaleManagerImpl<Type> extends ManagerBase implements AutoScale
     @Inject
     AutoScaleVmGroupPolicyMapDao _autoScaleVmGroupPolicyMapDao;
     @Inject
+    AutoScaleVmGroupVmMapDao _autoScaleVmGroupVmMapDao;
+    @Inject
     DataCenterDao _dcDao = null;
     @Inject
     UserDao _userDao;
@@ -145,6 +168,16 @@ public class AutoScaleManagerImpl<Type> extends ManagerBase implements AutoScale
     ConfigurationDao _configDao;
     @Inject
     IPAddressDao _ipAddressDao;
+    @Inject
+    AccountService _accountService;
+    @Inject
+    UserVmService _userVmService;
+    @Inject
+    UserVmManager _userVmManager;
+    @Inject 
+    LoadBalancerVMMapDao _LbVmMapDao;
+    @Inject 
+	LoadBalancingRulesService _LoadBalancingRulesService;
 
     public List<AutoScaleCounter> getSupportedAutoScaleCounters(long networkid) {
         String capability = _lbRulesMgr.getLBCapability(networkid, Capability.AutoScaleCounters.getName());
@@ -501,7 +534,7 @@ public class AutoScaleManagerImpl<Type> extends ManagerBase implements AutoScale
             throw new InvalidParameterValueException("action is invalid, only 'scaleup' and 'scaledown' is supported");
         }
 
-        AutoScalePolicyVO policyVO = new AutoScalePolicyVO(cmd.getDomainId(), cmd.getAccountId(), duration, quietTime, action);
+        AutoScalePolicyVO policyVO = new AutoScalePolicyVO(cmd.getDomainId(), cmd.getAccountId(), duration, quietTime, null, action);
 
         policyVO = checkValidityAndPersist(policyVO, cmd.getConditionIds());
         s_logger.info("Successfully created AutoScale Policy with Id: " + policyVO.getId());
@@ -705,9 +738,8 @@ public class AutoScaleManagerImpl<Type> extends ManagerBase implements AutoScale
                 "there are Vms already bound to the specified LoadBalancing Rule. User bound Vms and AutoScaled Vm Group cannot co-exist on a Load Balancing Rule");
         }
 
-        AutoScaleVmGroupVO vmGroupVO =
-            new AutoScaleVmGroupVO(cmd.getLbRuleId(), zoneId, loadBalancer.getDomainId(), loadBalancer.getAccountId(), minMembers, maxMembers,
-                loadBalancer.getDefaultPortStart(), interval, cmd.getProfileId(), AutoScaleVmGroup.State_New);
+        AutoScaleVmGroupVO vmGroupVO = new AutoScaleVmGroupVO(cmd.getLbRuleId(), zoneId, loadBalancer.getDomainId(), loadBalancer.getAccountId(), minMembers, maxMembers,
+                loadBalancer.getDefaultPortStart(), interval, null, cmd.getProfileId(), AutoScaleVmGroup.State_New);
 
         vmGroupVO = checkValidityAndPersist(vmGroupVO, cmd.getScaleUpPolicyIds(), cmd.getScaleDownPolicyIds());
         s_logger.info("Successfully created Autoscale Vm Group with Id: " + vmGroupVO.getId());
@@ -1169,4 +1201,266 @@ public class AutoScaleManagerImpl<Type> extends ManagerBase implements AutoScale
             s_logger.debug("Deleted " + count + " Conditions for account Id: " + accountId);
         }
     }
+
+	private boolean checkConditionUp(AutoScaleVmGroupVO asGroup, Integer numVm) {
+    	// check maximum
+		Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId());
+		Integer maxVm = asGroup.getMaxMembers();
+		if (currentVM + numVm > maxVm) {
+			s_logger.warn("number of VM will greater than the maximum in this group if scaling up, so do nothing more");
+			return false;
+		}
+		return true;
+    }
+	
+	private boolean checkConditionDown(AutoScaleVmGroupVO asGroup) {
+		Integer currentVM = _autoScaleVmGroupVmMapDao.countByGroup(asGroup.getId());
+		Integer minVm = asGroup.getMinMembers();
+		if (currentVM - 1 < minVm) {
+			s_logger.warn("number of VM will less than the minimum in this group if scaling down, so do nothing more");
+			return false;
+		}
+		return true;
+	}
+	
+	private long createNewVM(AutoScaleVmGroupVO asGroup) {
+		AutoScaleVmProfileVO profileVo = _autoScaleVmProfileDao.findById(asGroup.getProfileId());
+		long templateId = profileVo.getTemplateId();
+		long serviceOfferingId = profileVo.getServiceOfferingId();
+		if (templateId == -1) {
+			return -1;
+		}
+		// create new VM into DB
+		try {
+            //Verify that all objects exist before passing them to the service
+            Account owner = _accountService.getActiveAccountById(profileVo.getAccountId());
+
+            DataCenter zone = _entityMgr.findById(DataCenter.class, profileVo.getZoneId());
+            if (zone == null) {
+                throw new InvalidParameterValueException("Unable to find zone by id=" + profileVo.getZoneId());
+            }
+
+            ServiceOffering serviceOffering = _entityMgr.findById(ServiceOffering.class, serviceOfferingId);
+            if (serviceOffering == null) {
+                throw new InvalidParameterValueException("Unable to find service offering: " + serviceOfferingId);
+            }
+
+            VirtualMachineTemplate template = _entityMgr.findById(VirtualMachineTemplate.class, templateId);
+            // Make sure a valid template ID was specified
+            if (template == null) {
+                throw new InvalidParameterValueException("Unable to use template " + templateId);
+            }
+
+            if (!zone.isLocalStorageEnabled()) {
+                if (serviceOffering.getUseLocalStorage()) {
+                    throw new InvalidParameterValueException("Zone is not configured to use local storage but service offering " + serviceOffering.getName() + " uses it");
+                }
+            }
+
+            UserVm vm = null;
+			IpAddresses addrs = new IpAddresses(null, null);
+			if (zone.getNetworkType() == NetworkType.Basic) {
+                vm = _userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(),
+                		"autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, null, true, null, null, null, null);
+            } else {
+                if (zone.isSecurityGroupEnabled())  {
+                    vm = _userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, null, null,
+                            owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, null, true, null, null, null, null);
+
+                } else {
+                    vm = _userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, null, owner, "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(), "autoScaleVm-" + asGroup.getId() + "-" + getCurrentTimeStampString(),
+                            null, null, null, HypervisorType.XenServer, HTTPMethod.GET, null, null, null, addrs, true, null, null, null, null);
+
+                }
+            }
+
+            if (vm != null) {
+				return vm.getId();
+            } else {
+				return -1;
+            }
+        } catch (InsufficientCapacityException ex) {
+            s_logger.info(ex);
+            s_logger.trace(ex.getMessage(), ex);
+            throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, ex.getMessage());
+        } catch (ResourceUnavailableException ex) {
+            s_logger.warn("Exception: ", ex);
+            throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage());
+        }  catch (ConcurrentOperationException ex) {
+            s_logger.warn("Exception: ", ex);
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
+        } catch (ResourceAllocationException ex) {
+            s_logger.warn("Exception: ", ex);
+            throw new ServerApiException(ApiErrorCode.RESOURCE_ALLOCATION_ERROR, ex.getMessage());
+        }
+	}
+	
+	private String getCurrentTimeStampString(){
+		Date current = new Date();
+		SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
+		
+		return sdf.format(current);
+	}
+
+	private boolean startNewVM(long vmId) {
+		try {
+            CallContext.current().setEventDetails("Vm Id: "+vmId);
+			_userVmManager.startVirtualMachine(vmId, null, null);
+        } catch (ResourceUnavailableException ex) {
+            s_logger.warn("Exception: ", ex);
+            throw new ServerApiException(ApiErrorCode.RESOURCE_UNAVAILABLE_ERROR, ex.getMessage());
+        } catch (ConcurrentOperationException ex) {
+            s_logger.warn("Exception: ", ex);
+            throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage());
+        } catch (InsufficientCapacityException ex) {
+            StringBuilder message = new StringBuilder(ex.getMessage());
+            if (ex instanceof InsufficientServerCapacityException) {
+                if(((InsufficientServerCapacityException)ex).isAffinityApplied()){
+                    message.append(", Please check the affinity groups provided, there may not be sufficient capacity to follow them");
+                }
+            }
+            s_logger.info(ex);
+            s_logger.info(message.toString(), ex);
+            throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, message.toString());
+        }
+		return true;
+	}
+	
+	private boolean assignLBruleToNewVm(long vmId, AutoScaleVmGroupVO asGroup) {
+		List<Long> lstVmId = new ArrayList<Long>();
+    	long lbId = asGroup.getLoadBalancerId();
+    	
+    	List<LoadBalancerVMMapVO>  LbVmMapVos = _LbVmMapDao.listByLoadBalancerId(lbId);
+		if ((LbVmMapVos != null) && (LbVmMapVos.size() > 0)) {
+			for (LoadBalancerVMMapVO LbVmMapVo : LbVmMapVos) {
+				long instanceId = LbVmMapVo.getInstanceId();
+				if (instanceId == vmId) {
+					s_logger.warn("the new VM is already mapped to LB rule. What's wrong?");
+					return true;
+				}
+			}
+    	}
+		lstVmId.add(new Long(vmId));
+		return _LoadBalancingRulesService.assignToLoadBalancer(lbId, lstVmId);
+
+	}
+
+	private long removeLBrule(AutoScaleVmGroupVO asGroup) {
+    	long lbId = asGroup.getLoadBalancerId();
+		long instanceId = -1;
+    	List<LoadBalancerVMMapVO>  LbVmMapVos = _LbVmMapDao.listByLoadBalancerId(lbId);
+		if ((LbVmMapVos != null) && (LbVmMapVos.size() > 0)) {
+			for (LoadBalancerVMMapVO LbVmMapVo : LbVmMapVos) {
+				instanceId = LbVmMapVo.getInstanceId();
+			}
+    	}
+		// take last VM out of the list
+		List<Long> lstVmId = new ArrayList<Long>();
+		if (instanceId != -1)
+			lstVmId.add(instanceId);
+		if (_LoadBalancingRulesService.removeFromLoadBalancer(lbId, lstVmId))
+			return instanceId;
+		else
+			return -1;
+	}
+
+	@Override
+	public void doScaleUp(long groupId, Integer numVm) {
+		AutoScaleVmGroupVO asGroup = _autoScaleVmGroupDao.findById(groupId);
+		if (asGroup == null) {
+			s_logger.error("Can not find the groupid " + groupId + " for scaling up");
+			return;
+		}
+		if (!checkConditionUp(asGroup, numVm)) {
+			return;
+		}
+		for (int i = 0; i < numVm; i++) {
+			long vmId = createNewVM(asGroup);
+			if (vmId == -1) {
+				s_logger.error("Can not deploy new VM for scaling up in the group "
+						+ asGroup.getId() + ". Waiting for next round");
+				break;
+			}
+			if (startNewVM(vmId)) {
+				if (assignLBruleToNewVm(vmId, asGroup)) {
+					// persist to DB
+					AutoScaleVmGroupVmMapVO GroupVmVO = new AutoScaleVmGroupVmMapVO(
+							asGroup.getId(), vmId);
+					_autoScaleVmGroupVmMapDao.persist(GroupVmVO);
+					// update last_quiettime
+					List<AutoScaleVmGroupPolicyMapVO> GroupPolicyVOs = _autoScaleVmGroupPolicyMapDao
+							.listByVmGroupId(groupId);
+					for (AutoScaleVmGroupPolicyMapVO GroupPolicyVO : GroupPolicyVOs) {
+						AutoScalePolicyVO vo = _autoScalePolicyDao
+								.findById(GroupPolicyVO.getPolicyId());
+						if (vo.getAction().equals("scaleup")) {
+							vo.setLastQuiteTime(new Date());
+							_autoScalePolicyDao.persist(vo);
+							break;
+						}
+					}
+				} else {
+					s_logger.error("Can not assign LB rule for this new VM");
+					break;
+				}
+			} else {
+				s_logger.error("Can not deploy new VM for scaling up in the group "
+						+ asGroup.getId() + ". Waiting for next round");
+				break;
+			}
+		}
+	}
+
+	@Override
+	public void doScaleDown(final long groupId) {
+		AutoScaleVmGroupVO asGroup = _autoScaleVmGroupDao.findById(groupId);
+		if (asGroup == null) {
+			s_logger.error("Can not find the groupid " + groupId + " for scaling up");
+			return;
+		}
+		if (!checkConditionDown(asGroup)) {
+			return;
+		}
+		final long vmId = removeLBrule(asGroup);
+		if (vmId != -1) {
+			long profileId = asGroup.getProfileId();
+
+			// update group-vm mapping
+			_autoScaleVmGroupVmMapDao.remove(groupId, vmId);
+			// update last_quiettime
+			List<AutoScaleVmGroupPolicyMapVO> GroupPolicyVOs = _autoScaleVmGroupPolicyMapDao.listByVmGroupId(groupId);
+			for (AutoScaleVmGroupPolicyMapVO GroupPolicyVO : GroupPolicyVOs) {
+				AutoScalePolicyVO vo = _autoScalePolicyDao.findById(GroupPolicyVO.getPolicyId());
+				if (vo.getAction().equals("scaledown")) {
+					vo.setLastQuiteTime(new Date());
+					_autoScalePolicyDao.persist(vo);
+					break;
+				}
+			}
+
+			// get destroyvmgrace param
+			AutoScaleVmProfileVO asProfile = _autoScaleVmProfileDao.findById(profileId);
+			Integer destroyVmGracePeriod = asProfile.getDestroyVmGraceperiod();
+			if (destroyVmGracePeriod >= 0) {
+				_executor.schedule(new Runnable() {
+					@Override
+					public void run() {
+						try {
+
+							_userVmManager.destroyVm(vmId);
+
+						} catch (ResourceUnavailableException e) {
+							e.printStackTrace();
+						} catch (ConcurrentOperationException e) {
+							e.printStackTrace();
+						}
+					}
+				}, destroyVmGracePeriod, TimeUnit.SECONDS);
+			}
+		} else {
+			s_logger.error("Can not remove LB rule for the VM being destroyed. Do nothing more.");
+		}
+	}
+	
+
 }

server/src/com/cloud/network/element/VirtualRouterElement.java

@@ -30,11 +30,13 @@ import org.apache.log4j.Logger;
 import com.google.gson.Gson;
 
 import org.apache.cloudstack.api.command.admin.router.ConfigureOvsElementCmd;
+
 import org.apache.cloudstack.api.command.admin.router.ConfigureVirtualRouterElementCmd;
 import org.apache.cloudstack.api.command.admin.router.CreateVirtualRouterElementCmd;
 import org.apache.cloudstack.api.command.admin.router.ListOvsElementsCmd;
 import org.apache.cloudstack.api.command.admin.router.ListVirtualRouterElementsCmd;
 import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.to.LoadBalancerTO;
 import com.cloud.configuration.ConfigurationManager;
@@ -62,6 +64,8 @@ import com.cloud.network.RemoteAccessVpn;
 import com.cloud.network.VirtualRouterProvider;
 import com.cloud.network.VirtualRouterProvider.Type;
 import com.cloud.network.VpnUser;
+import com.cloud.network.as.AutoScaleCounter;
+import com.cloud.network.as.AutoScaleCounter.AutoScaleCounterType;
 import com.cloud.network.dao.IPAddressDao;
 import com.cloud.network.dao.LoadBalancerDao;
 import com.cloud.network.dao.NetworkDao;
@@ -99,6 +103,7 @@ import com.cloud.vm.VirtualMachine.State;
 import com.cloud.vm.VirtualMachineProfile;
 import com.cloud.vm.dao.DomainRouterDao;
 import com.cloud.vm.dao.UserVmDao;
+import com.google.gson.Gson;
 
 import com.google.gson.Gson;
 
@@ -112,8 +117,9 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl
         LoadBalancingServiceProvider, PortForwardingServiceProvider, RemoteAccessVPNServiceProvider, IpDeployer,
         NetworkMigrationResponder {
     private static final Logger s_logger = Logger.getLogger(VirtualRouterElement.class);
-
-    protected static final Map<Service, Map<Capability, String>> capabilities = setCapabilities();
+	public static final AutoScaleCounterType AutoScaleCounterCpu = new AutoScaleCounterType("cpu");
+	public static final AutoScaleCounterType AutoScaleCounterMemory = new AutoScaleCounterType("memory");
+	protected static final Map<Service, Map<Capability, String>> capabilities = setCapabilities();
 
     @Inject
     NetworkDao _networksDao;
@@ -240,6 +246,13 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl
                 return true;
             }
 
+            if (rules != null && rules.size() == 1 ) {
+                // for VR no need to add default egress rule to DENY traffic
+                if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System &&
+                        ! _networkMgr.getNetworkEgressDefaultPolicy(config.getId()))
+                    return true;
+            }
+
             if (!_routerMgr.applyFirewallRules(config, rules, routers)) {
                 throw new CloudRuntimeException("Failed to apply firewall rules in network " + config.getId());
             } else {
@@ -555,7 +568,17 @@ public class VirtualRouterElement extends AdapterBase implements VirtualRouterEl
         lbCapabilities.put(Capability.SupportedProtocols, "tcp, udp");
         lbCapabilities.put(Capability.SupportedStickinessMethods, getHAProxyStickinessCapability());
         lbCapabilities.put(Capability.LbSchemes, LoadBalancerContainer.Scheme.Public.toString());
-
+        
+        //specifies that LB rules can support autoscaling and the list of counters it supports
+        AutoScaleCounter counter;
+        List<AutoScaleCounter> counterList = new ArrayList<AutoScaleCounter>();
+		counter = new AutoScaleCounter(AutoScaleCounterCpu);
+        counterList.add(counter);
+		counter = new AutoScaleCounter(AutoScaleCounterMemory);
+        counterList.add(counter);
+        Gson gson = new Gson();
+        String autoScaleCounterList = gson.toJson(counterList);
+        lbCapabilities.put(Capability.AutoScaleCounters, autoScaleCounterList);
         capabilities.put(Service.Lb, lbCapabilities);
 
         // Set capabilities for Firewall service

server/src/com/cloud/network/firewall/FirewallManagerImpl.java

@@ -612,7 +612,6 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
     @Override
     public boolean applyEgressFirewallRules(FirewallRule rule, Account caller) throws ResourceUnavailableException {
         List<FirewallRuleVO> rules = _firewallDao.listByNetworkPurposeTrafficType(rule.getNetworkId(), Purpose.Firewall, FirewallRule.TrafficType.Egress);
-        applyDefaultEgressFirewallRule(rule.getNetworkId(), true);
         return applyFirewallRules(rules, false, caller);
     }
 
@@ -646,12 +645,8 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
     }
 
     @Override
-    public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy) throws ResourceUnavailableException {
+    public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy, boolean add) throws ResourceUnavailableException {
 
-        if (defaultPolicy == false) {
-            //If default policy is false no need apply rules on backend because firewall provider blocks by default
-            return true;
-        }
         s_logger.debug("applying default firewall egress rules ");
 
         NetworkVO network = _networkDao.findById(networkId);
@@ -661,6 +656,7 @@ public class FirewallManagerImpl extends ManagerBase implements FirewallService,
         FirewallRuleVO ruleVO =
             new FirewallRuleVO(null, null, null, null, "all", networkId, network.getAccountId(), network.getDomainId(), Purpose.Firewall, sourceCidr, null, null, null,
                 FirewallRule.TrafficType.Egress, FirewallRuleType.System);
+        ruleVO.setState(add ? State.Add : State.Revoke);
         List<FirewallRuleVO> rules = new ArrayList<FirewallRuleVO>();
         rules.add(ruleVO);
 

server/src/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java

@@ -915,7 +915,8 @@ public class LoadBalancingRulesManagerImpl<Type> extends ManagerBase implements
         if (provider == null || provider.size() == 0) {
             return false;
         }
-        if (provider.get(0) == Provider.Netscaler || provider.get(0) == Provider.F5BigIp) {
+        if (provider.get(0) == Provider.Netscaler || provider.get(0) == Provider.F5BigIp || 
+        		provider.get(0) == Provider.VirtualRouter) {
             return true;
         }
         return false;

server/src/com/cloud/server/StatsCollector.java

@@ -48,6 +48,7 @@ import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.GetStorageStatsCommand;
 import com.cloud.agent.api.HostStatsEntry;
+import com.cloud.agent.api.PerformanceMonitorCommand;
 import com.cloud.agent.api.VmDiskStatsEntry;
 import com.cloud.agent.api.VmStatsEntry;
 import com.cloud.cluster.ManagementServerHostVO;
@@ -59,8 +60,29 @@ import com.cloud.host.HostVO;
 import com.cloud.host.Status;
 import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.network.as.AutoScaleManager;
+import com.cloud.network.as.AutoScalePolicyConditionMapVO;
+import com.cloud.network.as.AutoScalePolicyVO;
+import com.cloud.network.as.AutoScaleVmGroupPolicyMapVO;
+import com.cloud.network.as.AutoScaleVmGroupVO;
+import com.cloud.network.as.AutoScaleVmGroupVmMapVO;
+import com.cloud.network.as.AutoScaleVmProfileVO;
+import com.cloud.network.as.Condition.Operator;
+import com.cloud.network.as.ConditionVO;
+import com.cloud.network.as.Counter;
+import com.cloud.network.as.CounterVO;
+import com.cloud.network.as.dao.AutoScalePolicyConditionMapDao;
+import com.cloud.network.as.dao.AutoScalePolicyDao;
+import com.cloud.network.as.dao.AutoScaleVmGroupDao;
+import com.cloud.network.as.dao.AutoScaleVmGroupPolicyMapDao;
+import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao;
+import com.cloud.network.as.dao.AutoScaleVmProfileDao;
+import com.cloud.network.as.dao.ConditionDao;
+import com.cloud.network.as.dao.CounterDao;
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
+import com.cloud.service.ServiceOfferingVO;
+import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StorageStats;
 import com.cloud.storage.VolumeStats;
@@ -70,6 +92,7 @@ import com.cloud.storage.dao.VolumeDao;
 import com.cloud.user.VmDiskStatisticsVO;
 import com.cloud.user.dao.VmDiskStatisticsDao;
 import com.cloud.utils.NumbersUtil;
+import com.cloud.utils.Pair;
 import com.cloud.utils.component.ComponentMethodInterceptable;
 import com.cloud.utils.component.ManagerBase;
 import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -82,9 +105,11 @@ import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.net.MacAddress;
 import com.cloud.vm.UserVmManager;
 import com.cloud.vm.UserVmVO;
+import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VmStats;
 import com.cloud.vm.dao.UserVmDao;
+import com.cloud.vm.dao.VMInstanceDao;
 
 /**
  * Provides real time stats for various agent resources up to x seconds
@@ -92,55 +117,53 @@ import com.cloud.vm.dao.UserVmDao;
  */
 @Component
 public class StatsCollector extends ManagerBase implements ComponentMethodInterceptable {
-    public static final Logger s_logger = Logger.getLogger(StatsCollector.class.getName());
 
-    private static StatsCollector s_instance = null;
+	public static final Logger s_logger = Logger.getLogger(StatsCollector.class.getName());
 
-    private ScheduledExecutorService _executor = null;
-    @Inject
-    private AgentManager _agentMgr;
-    @Inject
-    private UserVmManager _userVmMgr;
-    @Inject
-    private HostDao _hostDao;
-    @Inject
-    private UserVmDao _userVmDao;
-    @Inject
-    private VolumeDao _volsDao;
-    @Inject
-    private PrimaryDataStoreDao _storagePoolDao;
-    @Inject
-    private ImageStoreDao _imageStoreDao;
-    @Inject
-    private StorageManager _storageManager;
-    @Inject
-    private StoragePoolHostDao _storagePoolHostDao;
-    @Inject
-    private DataStoreManager _dataStoreMgr;
-    @Inject
-    private ResourceManager _resourceMgr;
-    @Inject
-    private ConfigurationDao _configDao;
-    @Inject
-    private EndPointSelector _epSelector;
-    @Inject
-    private VmDiskStatisticsDao _vmDiskStatsDao;
-    @Inject
-    private ManagementServerHostDao _msHostDao;
+	private static StatsCollector s_instance = null;
 
-    private ConcurrentHashMap<Long, HostStats> _hostStats = new ConcurrentHashMap<Long, HostStats>();
-    private final ConcurrentHashMap<Long, VmStats> _VmStats = new ConcurrentHashMap<Long, VmStats>();
-    private ConcurrentHashMap<Long, VolumeStats> _volumeStats = new ConcurrentHashMap<Long, VolumeStats>();
-    private ConcurrentHashMap<Long, StorageStats> _storageStats = new ConcurrentHashMap<Long, StorageStats>();
-    private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new ConcurrentHashMap<Long, StorageStats>();
+	private ScheduledExecutorService _executor = null;
+	@Inject private AgentManager _agentMgr;
+	@Inject private UserVmManager _userVmMgr;
+	@Inject private HostDao _hostDao;
+	@Inject private UserVmDao _userVmDao;
+	@Inject private VolumeDao _volsDao;
+	@Inject private PrimaryDataStoreDao _storagePoolDao;
+	@Inject private ImageStoreDao _imageStoreDao;
+	@Inject private StorageManager _storageManager;
+	@Inject private StoragePoolHostDao _storagePoolHostDao;
+	@Inject private DataStoreManager _dataStoreMgr;
+	@Inject private ResourceManager _resourceMgr;
+    @Inject private ConfigurationDao _configDao;
+    @Inject private EndPointSelector _epSelector;
+    @Inject private VmDiskStatisticsDao _vmDiskStatsDao;
+    @Inject private ManagementServerHostDao _msHostDao;
+	@Inject	private AutoScaleVmGroupDao _asGroupDao;
+	@Inject private AutoScaleVmGroupVmMapDao _asGroupVmDao;
+	@Inject private AutoScaleManager _asManager;
+	@Inject private VMInstanceDao _vmInstance;
+	@Inject private AutoScaleVmGroupPolicyMapDao _asGroupPolicyDao;
+	@Inject private AutoScalePolicyDao _asPolicyDao;
+	@Inject private AutoScalePolicyConditionMapDao _asConditionMapDao; 
+	@Inject private ConditionDao _asConditionDao;
+	@Inject private CounterDao _asCounterDao;
+	@Inject private AutoScaleVmProfileDao _asProfileDao;
+	@Inject private ServiceOfferingDao _serviceOfferingDao;
 
-    long hostStatsInterval = -1L;
-    long hostAndVmStatsInterval = -1L;
-    long storageStatsInterval = -1L;
-    long volumeStatsInterval = -1L;
-    int vmDiskStatsInterval = 0;
+	private ConcurrentHashMap<Long, HostStats> _hostStats = new ConcurrentHashMap<Long, HostStats>();
+	private final ConcurrentHashMap<Long, VmStats> _VmStats = new ConcurrentHashMap<Long, VmStats>();
+	private ConcurrentHashMap<Long, VolumeStats> _volumeStats = new ConcurrentHashMap<Long, VolumeStats>();
+	private ConcurrentHashMap<Long, StorageStats> _storageStats = new ConcurrentHashMap<Long, StorageStats>();
+	private ConcurrentHashMap<Long, StorageStats> _storagePoolStats = new ConcurrentHashMap<Long, StorageStats>();
 
-    private ScheduledExecutorService _diskStatsUpdateExecutor;
+	long hostStatsInterval = -1L;
+	long hostAndVmStatsInterval = -1L;
+	long storageStatsInterval = -1L;
+	long volumeStatsInterval = -1L;
+	long autoScaleStatsInterval = -1L;
+	int vmDiskStatsInterval = 0;
+
+	private ScheduledExecutorService _diskStatsUpdateExecutor;
     private int _usageAggregationRange = 1440;
     private String _usageTimeZone = "GMT";
     private final long mgmtSrvrId = MacAddress.getMacAddress().toLong();
@@ -169,14 +192,15 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
         return true;
     }
 
-    private void init(Map<String, String> configs) {
-        _executor = Executors.newScheduledThreadPool(3, new NamedThreadFactory("StatsCollector"));
+	private void init(Map<String, String> configs) {
+		_executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("StatsCollector"));
 
-        hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L);
-        hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L);
-        storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L);
-        volumeStatsInterval = NumbersUtil.parseLong(configs.get("volume.stats.interval"), -1L);
-        vmDiskStatsInterval = NumbersUtil.parseInt(configs.get("vm.disk.stats.interval"), 0);
+		 hostStatsInterval = NumbersUtil.parseLong(configs.get("host.stats.interval"), 60000L);
+		 hostAndVmStatsInterval = NumbersUtil.parseLong(configs.get("vm.stats.interval"), 60000L);
+		 storageStatsInterval = NumbersUtil.parseLong(configs.get("storage.stats.interval"), 60000L);
+		 volumeStatsInterval = NumbersUtil.parseLong(configs.get("volume.stats.interval"), -1L);
+		 autoScaleStatsInterval = NumbersUtil.parseLong(configs.get("autoscale.stats.interval"), 60000L);
+		 vmDiskStatsInterval = NumbersUtil.parseInt(configs.get("vm.disk.stats.interval"), 0);
 
         if (hostStatsInterval > 0) {
             _executor.scheduleWithFixedDelay(new HostCollector(), 15000L, hostStatsInterval, TimeUnit.MILLISECONDS);
@@ -186,9 +210,13 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
             _executor.scheduleWithFixedDelay(new VmStatsCollector(), 15000L, hostAndVmStatsInterval, TimeUnit.MILLISECONDS);
         }
 
-        if (storageStatsInterval > 0) {
-            _executor.scheduleWithFixedDelay(new StorageCollector(), 15000L, storageStatsInterval, TimeUnit.MILLISECONDS);
-        }
+		if (storageStatsInterval > 0) {
+		    _executor.scheduleWithFixedDelay(new StorageCollector(), 15000L, storageStatsInterval, TimeUnit.MILLISECONDS);
+		}
+		 
+		if (autoScaleStatsInterval > 0) {
+            _executor.scheduleWithFixedDelay(new AutoScaleMonitor(), 15000L, autoScaleStatsInterval, TimeUnit.MILLISECONDS);
+		}
 
         if (vmDiskStatsInterval > 0) {
             if (vmDiskStatsInterval < 300)
@@ -578,21 +606,300 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc
                     }
                 }
                 _storagePoolStats = storagePoolStats;
-            } catch (Throwable t) {
-                s_logger.error("Error trying to retrieve storage stats", t);
-            }
-        }
-    }
+			} catch (Throwable t) {
+				s_logger.error("Error trying to retrieve storage stats", t);
+			}
+		}
+	}
 
-    public StorageStats getStorageStats(long id) {
-        return _storageStats.get(id);
-    }
+	class AutoScaleMonitor extends ManagedContextRunnable {
+		@Override
+		protected void runInContext() {
+			try {
+				if (s_logger.isDebugEnabled()) {
+					s_logger.debug("AutoScaling Monitor is running...");
+				}
+				// list all AS VMGroups
+				List<AutoScaleVmGroupVO> asGroups = _asGroupDao.listAll();
+				for (AutoScaleVmGroupVO asGroup : asGroups) {
+					// check group state
+					if ((asGroup.getState().equals("enabled")) && (is_native(asGroup.getId()))) {
+						// check minimum vm of group
+						Integer currentVM = _asGroupVmDao.countByGroup(asGroup.getId());
+						if (currentVM < asGroup.getMinMembers()) {
+							_asManager.doScaleUp(asGroup.getId(), asGroup.getMinMembers() - currentVM);
+							continue;
+						}
+						
+						//check interval
+						long now = (new Date()).getTime();
+						if (asGroup.getLastInterval() != null)
+							if ((now - asGroup.getLastInterval().getTime()) < (long) asGroup
+									.getInterval()) {
+								continue;
+							}
 
-    public HostStats getHostStats(long hostId) {
-        return _hostStats.get(hostId);
-    }
+						// update last_interval
+						asGroup.setLastInterval(new Date());
+						_asGroupDao.persist(asGroup);
 
-    public StorageStats getStoragePoolStats(long id) {
-        return _storagePoolStats.get(id);
-    }
+						// collect RRDs data for this group
+						if (s_logger.isDebugEnabled()) {
+							s_logger.debug("[AutoScale] Collecting RRDs data...");
+						}
+						Map<String, String> params = new HashMap<String, String>();
+						List <AutoScaleVmGroupVmMapVO> asGroupVmVOs = _asGroupVmDao.listByGroup(asGroup.getId());
+						params.put("total_vm", String.valueOf(asGroupVmVOs.size()));
+						for (int i = 0; i < asGroupVmVOs.size(); i++) {
+							long vmId = asGroupVmVOs.get(i).getInstanceId();
+							VMInstanceVO vmVO = _vmInstance.findById(vmId);
+							//xe vm-list | grep vmname -B 1 | head -n 1 | awk -F':' '{print $2}'
+							params.put("vmname" + String.valueOf(i + 1), vmVO.getInstanceName());
+							params.put("vmid" + String.valueOf(i + 1), String.valueOf(vmVO.getId()));
+
+						}
+						// get random hostid because all vms are in a cluster
+						long vmId = asGroupVmVOs.get(0).getInstanceId();
+						VMInstanceVO vmVO = _vmInstance.findById(vmId);
+						Long receiveHost = vmVO.getHostId();
+
+						// setup parameters phase: duration and counter
+						// list pair [counter, duration]
+						List<Pair<String, Integer>> lstPair = getPairofCounternameAndDuration(asGroup.getId());
+						int total_counter = 0;
+						String[] lstCounter = new String[lstPair.size()];
+						for (int i = 0; i < lstPair.size(); i++) {
+							Pair<String, Integer> pair = lstPair.get(i);
+							String strCounterNames = pair.first();
+							Integer duration = pair.second();
+							
+							lstCounter[i] = strCounterNames.split(",")[0];
+							total_counter++;
+							params.put("duration" + String.valueOf(total_counter), duration.toString());
+							params.put("counter" + String.valueOf(total_counter), lstCounter[i]);
+							params.put("con" + String.valueOf(total_counter), strCounterNames.split(",")[1]);
+						}
+						params.put("total_counter", String.valueOf(total_counter));
+
+						PerformanceMonitorCommand perfMon = new PerformanceMonitorCommand(params, 20);
+
+						try {
+							Answer answer = _agentMgr.send(receiveHost, perfMon);
+							if (answer == null || !answer.getResult()) {
+								s_logger.debug("Failed to send data to node !");
+							} else {
+								String result = answer.getDetails();
+								s_logger.debug("[AutoScale] RRDs collection answer: " + result);
+								HashMap<Long, Double> avgCounter = new HashMap<Long, Double>();
+								
+								// extract data
+								String[] counterElements = result.split(",");
+								if ((counterElements != null) && (counterElements.length > 0)) {
+									for (String string : counterElements) {
+										try {
+											String[] counterVals = string.split(":");
+											String[] counter_vm = counterVals[0].split("\\.");
+											
+											Long counterId = Long.parseLong(counter_vm[1]);
+											Long conditionId = Long.parseLong(params.get("con" + counter_vm[1]));
+											Double coVal = Double.parseDouble(counterVals[1]);
+
+											// Summary of all counter by counterId key
+											if (avgCounter.get(counterId) == null){													
+												/* initialize if data is not set */
+												avgCounter.put(counterId, new Double(0));
+											}
+											
+											String counterName = getCounternamebyCondition(conditionId.longValue());
+											if (counterName == Counter.Source.memory.toString()) {
+												// calculate memory in percent
+												Long profileId = asGroup.getProfileId();
+												AutoScaleVmProfileVO profileVo = _asProfileDao.findById(profileId);
+												ServiceOfferingVO serviceOff = _serviceOfferingDao.findById(profileVo.getServiceOfferingId());
+												int maxRAM = serviceOff.getRamSize();
+
+												// get current RAM percent
+												coVal = coVal / maxRAM;
+											} else {
+												// cpu
+												coVal = coVal * 100;
+											}
+											
+											// update data entry
+											avgCounter.put(counterId, avgCounter.get(counterId)+ coVal);
+											
+										} catch (Exception e) {
+											e.printStackTrace();
+										}
+									}
+
+									String scaleAction = getAutoscaleAction(avgCounter, asGroup.getId(), currentVM, params);
+									if (scaleAction != null) {
+										s_logger.debug("[AutoScale] Doing scale action: " + scaleAction + " for group " + asGroup.getId());
+										if (scaleAction.equals("scaleup")) {
+											_asManager.doScaleUp(asGroup.getId(), 1);
+										} else {
+											_asManager.doScaleDown(asGroup.getId());
+										}
+									}
+								}
+							}
+
+						} catch (Exception e) {
+							e.printStackTrace();
+						}
+
+					}
+				}
+
+			} catch (Throwable t) {
+				s_logger.error("Error trying to monitor autoscaling", t);
+			}
+
+		}
+		
+		private boolean is_native(long groupId) {
+			List <AutoScaleVmGroupPolicyMapVO> vos = _asGroupPolicyDao.listByVmGroupId(groupId);
+			for (AutoScaleVmGroupPolicyMapVO vo : vos) {
+				List<AutoScalePolicyConditionMapVO> ConditionPolicies = _asConditionMapDao.findByPolicyId(vo.getPolicyId());
+				for (AutoScalePolicyConditionMapVO ConditionPolicy : ConditionPolicies) {
+					ConditionVO condition = _asConditionDao.findById(ConditionPolicy.getConditionId());
+					CounterVO counter = _asCounterDao.findById(condition.getCounterid());
+					if (counter.getSource() == Counter.Source.cpu || counter.getSource() == Counter.Source.memory)
+						return true;
+				}
+			}
+			return false;
+		}
+		
+		private String getAutoscaleAction(HashMap<Long, Double> avgCounter, long groupId, long currentVM, Map<String, String> params) {
+			
+			List<AutoScaleVmGroupPolicyMapVO> listMap = _asGroupPolicyDao.listByVmGroupId(groupId);
+			if ( (listMap == null) || (listMap.size() == 0) )
+				return null;
+			for (AutoScaleVmGroupPolicyMapVO asVmgPmap : listMap) {
+				AutoScalePolicyVO policyVO = _asPolicyDao.findById(asVmgPmap.getPolicyId());
+				if (policyVO != null) {
+					Integer quitetime = policyVO.getQuietTime();
+					Date quitetimeDate = policyVO.getLastQuiteTime();
+					long last_quitetime = 0L;
+					if (quitetimeDate != null) {
+						last_quitetime = policyVO.getLastQuiteTime().getTime();
+					}
+					long current_time = (new Date()).getTime();
+
+					// check quite time for this policy
+					if ((current_time - last_quitetime) >= (long) quitetime) {
+
+						// list all condition of this policy
+						boolean bValid = true;
+						List<ConditionVO> lstConditions = getConditionsbyPolicyId(policyVO.getId());
+						if ((lstConditions != null) && (lstConditions.size() > 0)) {
+							// check whole conditions of this policy
+							for (ConditionVO conditionVO : lstConditions) {
+								long thresholdValue = conditionVO.getThreshold();
+								Double thresholdPercent = (double)thresholdValue / 100;
+								CounterVO counterVO = _asCounterDao.findById(conditionVO.getCounterid());
+//								Double sum = avgCounter.get(conditionVO.getCounterid());
+								long counter_count=1;
+								do {
+									String counter_param = params.get("counter" + String.valueOf(counter_count));
+									Counter.Source counter_source = counterVO.getSource(); 
+									if (counter_param.equals(counter_source.toString()))
+										break;
+									counter_count++;
+								} while (1==1);
+								
+								Double sum = avgCounter.get(counter_count);
+								Double avg = sum / currentVM;
+								Operator op = conditionVO.getRelationalOperator();
+								boolean bConditionCheck = ((op == com.cloud.network.as.Condition.Operator.EQ) && (thresholdPercent == avg))
+										|| ((op == com.cloud.network.as.Condition.Operator.GE) && (avg >= thresholdPercent))
+										|| ((op == com.cloud.network.as.Condition.Operator.GT) && (avg > thresholdPercent))
+										|| ((op == com.cloud.network.as.Condition.Operator.LE) && (avg <= thresholdPercent))
+										|| ((op == com.cloud.network.as.Condition.Operator.LT) && (avg < thresholdPercent));
+
+								if (!bConditionCheck) {
+									bValid = false;
+									break;
+								}
+							}
+							if (bValid) {
+								return policyVO.getAction();
+							}
+						}
+					}
+				}
+			}
+			return null;
+		}
+		
+		private List<ConditionVO> getConditionsbyPolicyId(long policyId){
+			List<AutoScalePolicyConditionMapVO> conditionMap = _asConditionMapDao.findByPolicyId(policyId);
+			if ( (conditionMap == null) || (conditionMap.size() == 0))
+				return null;
+			
+			List<ConditionVO> lstResult = new ArrayList<ConditionVO>();
+			for (AutoScalePolicyConditionMapVO asPCmap : conditionMap) {
+				lstResult.add(_asConditionDao.findById(asPCmap.getConditionId()));
+			}
+			
+			return lstResult;
+		}
+
+		public List<Pair<String, Integer>> getPairofCounternameAndDuration(
+				long groupId) {
+			AutoScaleVmGroupVO groupVo = _asGroupDao.findById(groupId);
+			if (groupVo == null)
+				return null;
+			List<Pair<String, Integer>> result = new ArrayList<Pair<String, Integer>>();
+			//list policy map
+			List<AutoScaleVmGroupPolicyMapVO> groupPolicymap = _asGroupPolicyDao.listByVmGroupId(groupVo.getId());
+			if (groupPolicymap == null)
+				return null;
+			for (AutoScaleVmGroupPolicyMapVO gpMap : groupPolicymap) {
+				//get duration
+				AutoScalePolicyVO policyVo = _asPolicyDao.findById(gpMap.getPolicyId());				
+				Integer duration = policyVo.getDuration();
+				//get collection of counter name
+				String counterNames = "";
+				List<AutoScalePolicyConditionMapVO> lstPCmap = _asConditionMapDao.findByPolicyId(policyVo.getId());
+				for (AutoScalePolicyConditionMapVO pcMap : lstPCmap) {
+					String counterName = getCounternamebyCondition(pcMap.getConditionId());
+					
+					counterNames += counterName + "," + pcMap.getConditionId();
+				}
+				// add to result
+				Pair<String, Integer> pair = new Pair<String, Integer>(counterNames, duration);
+				result.add(pair);
+			}
+
+			return result;
+		}
+		
+		public String getCounternamebyCondition(long conditionId){
+			
+			ConditionVO condition = _asConditionDao.findById(conditionId);
+			if (condition == null)
+				return "";
+			
+			long counterId = condition.getCounterid();
+			CounterVO counter = _asCounterDao.findById(counterId);
+			if (counter == null)
+				return "";
+			
+			return counter.getSource().toString();
+		}
+	}
+	public StorageStats getStorageStats(long id) {
+		return _storageStats.get(id);
+	}
+
+	public HostStats getHostStats(long hostId){
+		return _hostStats.get(hostId);
+	}
+
+	public StorageStats getStoragePoolStats(long id) {
+		return _storagePoolStats.get(id);
+	}
 }

server/src/com/cloud/vm/UserVmManagerImpl.java

@@ -36,6 +36,8 @@ import javax.naming.ConfigurationException;
 
 import com.cloud.event.UsageEventVO;
 import com.cloud.uuididentity.UUIDManager;
+import com.cloud.capacity.Capacity;
+import com.cloud.exception.InsufficientServerCapacityException;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.log4j.Logger;
 
@@ -84,6 +86,8 @@ import org.apache.cloudstack.managed.context.ManagedContextRunnable;
 import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.AgentManager;
 import com.cloud.agent.api.Answer;
@@ -1306,6 +1310,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         int currentCpu = currentServiceOffering.getCpu();
         int currentMemory = currentServiceOffering.getRamSize();
         int currentSpeed = currentServiceOffering.getSpeed();
+        int memoryDiff = newMemory - currentMemory;
+        int cpuDiff =  newCpu*newSpeed - currentCpu*currentSpeed;
 
         // Don't allow to scale when (Any of the new values less than current values) OR (All current and new values are same)
         if ((newSpeed < currentSpeed || newMemory < currentMemory || newCpu < currentCpu) ||
@@ -1328,14 +1334,24 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
         if (vmInstance.getState().equals(State.Running)) {
             int retry = _scaleRetry;
             ExcludeList excludes = new ExcludeList();
+
+            // Check zone wide flag
             boolean enableDynamicallyScaleVm = EnableDynamicallyScaleVm.valueIn(vmInstance.getDataCenterId());
             if (!enableDynamicallyScaleVm) {
                 throw new PermissionDeniedException("Dynamically scaling virtual machines is disabled for this zone, please contact your admin");
             }
+
+            // Check vm flag
             if (!vmInstance.isDynamicallyScalable()) {
                 throw new CloudRuntimeException("Unable to Scale the vm: " + vmInstance.getUuid() + " as vm does not have tools to support dynamic scaling");
             }
 
+            // Check disable threshold for cluster is not crossed
+            HostVO host = _hostDao.findById(vmInstance.getHostId());
+            if(_capacityMgr.checkIfClusterCrossesThreshold(host.getClusterId(), cpuDiff, memoryDiff)){
+                throw new CloudRuntimeException("Unable to scale vm: " + vmInstance.getUuid() + " due to insufficient resources");
+            }
+
             while (retry-- != 0) { // It's != so that it can match -1.
                 try {
                     boolean existingHostHasCapacity = false;
@@ -1344,15 +1360,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                     if (newCpu > currentCpu) {
                         _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu));
                     }
-                    if (newMemory > currentMemory) {
-                        _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory));
+
+                    if (memoryDiff > 0) {
+                        _resourceLimitMgr.incrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long (memoryDiff));
                     }
 
                     // #1 Check existing host has capacity
                     if( !excludes.shouldAvoid(ApiDBUtils.findHostById(vmInstance.getHostId())) ){
                         existingHostHasCapacity = _capacityMgr.checkIfHostHasCpuCapability(vmInstance.getHostId(), newCpu, newSpeed)
-                                && _capacityMgr.checkIfHostHasCapacity(vmInstance.getHostId(), newServiceOffering.getSpeed() - currentServiceOffering.getSpeed(),
-                                (newServiceOffering.getRamSize() - currentServiceOffering.getRamSize()) * 1024L * 1024L, false, ApiDBUtils.getCpuOverprovisioningFactor(), 1f, false); // TO DO fill it with mem.
+                                && _capacityMgr.checkIfHostHasCapacity(vmInstance.getHostId(), cpuDiff,
+                                (memoryDiff) * 1024L * 1024L, false, _capacityMgr.getClusterOverProvisioningFactor(host.getClusterId(), Capacity.CAPACITY_TYPE_CPU),
+                                _capacityMgr.getClusterOverProvisioningFactor(host.getClusterId(), Capacity.CAPACITY_TYPE_MEMORY), false);
                         excludes.addHost(vmInstance.getHostId());
                     }
 
@@ -1392,8 +1410,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
                         if (newCpu > currentCpu) {
                             _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.cpu, new Long(newCpu - currentCpu));
                         }
-                        if (newMemory > currentMemory) {
-                            _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long(newMemory - currentMemory));
+
+                        if (memoryDiff > 0) {
+                            _resourceLimitMgr.decrementResourceCount(caller.getAccountId(), ResourceType.memory, new Long (memoryDiff));
                         }
                     }
                 }
@@ -3345,9 +3364,12 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
     }
 
     @Override
-    public UserVm destroyVm(long vmId) throws ResourceUnavailableException, ConcurrentOperationException {
-        Account caller = CallContext.current().getCallingAccount();
-        Long userId = CallContext.current().getCallingUserId();
+
+    public UserVm destroyVm(long vmId) throws ResourceUnavailableException,
+    ConcurrentOperationException {
+		// Account caller = CallContext.current().getCallingAccount();
+		// Long userId = CallContext.current().getCallingUserId();
+		Long userId = 2L;
 
         // Verify input parameters
         UserVmVO vm = _vmDao.findById(vmId);
@@ -3361,8 +3383,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir
             return vm;
         }
 
-        _accountMgr.checkAccess(caller, null, true, vm);
-        _userDao.findById(userId);
+
+		// _accountMgr.checkAccess(caller, null, true, vm);
+        User userCaller = _userDao.findById(userId);
 
         boolean status;
         State vmState = vm.getState();

server/test/com/cloud/network/MockFirewallManagerImpl.java

@@ -149,7 +149,7 @@ public class MockFirewallManagerImpl extends ManagerBase implements FirewallMana
     }
 
     @Override
-    public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy) throws ResourceUnavailableException {
+    public boolean applyDefaultEgressFirewallRule(Long networkId, boolean defaultPolicy, boolean add) throws ResourceUnavailableException {
         return false;  //To change body of implemented methods use File | Settings | File Templates.
     }
 

server/test/com/cloud/network/MockNetworkModelImpl.java

@@ -864,4 +864,9 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel {
     public boolean isNetworkReadyForGc(long networkId) {
         return true;
     }
+
+    @Override
+    public boolean getNetworkEgressDefaultPolicy(Long networkId) {
+        return false;  //To change body of implemented methods use File | Settings | File Templates.
+    }
 }

server/test/com/cloud/vpc/MockNetworkModelImpl.java

@@ -878,4 +878,9 @@ public class MockNetworkModelImpl extends ManagerBase implements NetworkModel {
     public boolean isNetworkReadyForGc(long networkId) {
         return true;
     }
+
+    @Override
+    public boolean getNetworkEgressDefaultPolicy(Long networkId) {
+        return false;  //To change body of implemented methods use File | Settings | File Templates.
+    }
 }

setup/db/create-schema.sql

@@ -152,6 +152,7 @@ DROP TABLE IF EXISTS `cloud`.`template_s3_ref`;
 DROP TABLE IF EXISTS `cloud`.`nicira_nvp_router_map`;
 DROP TABLE IF EXISTS `cloud`.`external_bigswitch_vns_devices`;
 DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroup_policy_map`;
+DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroup_vm_map`;
 DROP TABLE IF EXISTS `cloud`.`autoscale_policy_condition_map`;
 DROP TABLE IF EXISTS `cloud`.`autoscale_vmgroups`;
 DROP TABLE IF EXISTS `cloud`.`autoscale_vmprofiles`;

setup/db/db/schema-40to410.sql

@@ -345,6 +345,7 @@ CREATE TABLE `cloud`.`autoscale_policies` (
   `account_id` bigint unsigned NOT NULL,
   `duration` int unsigned NOT NULL,
   `quiet_time` int unsigned NOT NULL,
+  `last_quiet_time` datetime DEFAULT NULL,
   `action` varchar(15),
   `created` datetime NOT NULL COMMENT 'date created',
   `removed` datetime COMMENT 'date removed if not null',
@@ -366,6 +367,7 @@ CREATE TABLE `cloud`.`autoscale_vmgroups` (
   `max_members` int unsigned NOT NULL,
   `member_port` int unsigned NOT NULL,
   `interval` int unsigned NOT NULL,
+  `last_interval` datetime DEFAULT NULL,
   `profile_id` bigint unsigned NOT NULL,
   `state` varchar(255) NOT NULL COMMENT 'enabled or disabled, a vmgroup is disabled to stop autoscaling activity',
   `created` datetime NOT NULL COMMENT 'date created',
@@ -401,10 +403,22 @@ CREATE TABLE `cloud`.`autoscale_vmgroup_policy_map` (
   INDEX `i_autoscale_vmgroup_policy_map__vmgroup_id`(`vmgroup_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
+CREATE TABLE `cloud`.`autoscale_vmgroup_vm_map` (
+  `id` bigint unsigned NOT NULL auto_increment,
+  `vmgroup_id` bigint unsigned NOT NULL,
+  `instance_id` bigint unsigned NOT NULL,
+  PRIMARY KEY  (`id`),
+  CONSTRAINT `fk_autoscale_vmgroup_vm_map__vmgroup_id` FOREIGN KEY `fk_autoscale_vmgroup_vm_map__vmgroup_id` (`vmgroup_id`) REFERENCES `autoscale_vmgroups` (`id`) ON DELETE CASCADE,
+  CONSTRAINT `fk_autoscale_vmgroup_vm_map__instance_id` FOREIGN KEY `fk_autoscale_vmgroup_vm_map__instance_id` (`instance_id`) REFERENCES `vm_instance` (`id`),
+  INDEX `i_autoscale_vmgroup_vm_map__vmgroup_id`(`vmgroup_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
 INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (1, UUID(), 'snmp','Linux User CPU - percentage', '1.3.6.1.4.1.2021.11.9.0', now());
 INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (2, UUID(), 'snmp','Linux System CPU - percentage', '1.3.6.1.4.1.2021.11.10.0', now());
 INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (3, UUID(), 'snmp','Linux CPU Idle - percentage', '1.3.6.1.4.1.2021.11.11.0', now());
 INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (100, UUID(), 'netscaler','Response Time - microseconds', 'RESPTIME', now());
+INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (4, UUID(), 'cpu','Linux User CPU - percentage - native', '1.3.6.1.4.1.2021.11.9.1', now());
+INSERT INTO `cloud`.`counter` (id, uuid, source, name, value,created) VALUES (5, UUID(), 'memory','Linux User RAM - percentage - native', '1.3.6.1.4.1.2021.11.10.1', now());
 
 CREATE TABLE  `cloud`.`user_ipv6_address` (
   `id` bigint unsigned NOT NULL UNIQUE auto_increment,

setup/db/db/schema-421to430.sql

@@ -804,3 +804,12 @@ CREATE TABLE `cloud`.`network_acl_item_details` (
 ALTER TABLE `cloud`.`alert` ADD COLUMN `name` varchar(255) DEFAULT NULL COMMENT 'name of the alert';
 
 UPDATE `cloud`.`hypervisor_capabilities` SET `max_data_volumes_limit`=13 WHERE `hypervisor_type`='Vmware';
+
+ALTER TABLE `cloud`.`external_load_balancer_devices` ADD COLUMN `is_exclusive_gslb_provider` int(1) unsigned NOT NULL DEFAULT 0 COMMENT '1 if load balancer appliance is acting exclusively as gslb service provider in the zone and can not be used for LB';
+
+DELETE FROM `cloud`.`configuration` WHERE `name` IN ("xen.update.url", "update.check.interval", "baremetal_dhcp_devices", "host.updates.enable");
+
+INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'VMSnapshotManager', "vmsnapshot.create.wait", "1800", "In second, timeout for create vm snapshot", NULL, NULL,NULL,0);
+INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Advanced", 'DEFAULT', 'VMSnapshotManager', "vmsnapshot.max", "10", "Maximum vm snapshots for a vm", NULL, NULL,NULL,0);
+
+UPDATE `cloud`.`configuration` SET `component` = 'VMSnapshotManager' WHERE `name` IN ("vmsnapshot.create.wait", "vmsnapshot.max");

systemvm/patches/debian/vpn/opt/cloud/bin/vpn_l2tp.sh

@@ -35,21 +35,41 @@ get_intf_ip() {
 iptables_() {
    local op=$1
    local public_ip=$2
+   local is_vpc=false
+   local forward_action="ACCEPT"
+   if grep "vpcrouter" /var/cache/cloud/cmdline &> /dev/null
+   then
+	is_vpc=true
+   fi
 
    sudo iptables $op INPUT -i $dev --dst $public_ip -p udp -m udp --dport 500 -j ACCEPT
    sudo iptables $op INPUT -i $dev --dst $public_ip -p udp -m udp --dport 4500 -j ACCEPT
    sudo iptables $op INPUT -i $dev --dst $public_ip -p udp -m udp --dport 1701 -j ACCEPT
    sudo iptables $op INPUT -i $dev -p ah -j ACCEPT
    sudo iptables $op INPUT -i $dev -p esp -j ACCEPT
-   sudo iptables $op FORWARD -i ppp+ -d $cidr -j ACCEPT
-   sudo iptables $op FORWARD -s $cidr -o ppp+ -j ACCEPT
-   sudo iptables $op FORWARD -i ppp+ -o ppp+ -j ACCEPT
+   if $is_vpc
+   then
+       # Need to apply the following ACL rules as well.
+       if sudo iptables -N VPN_FORWARD &> /dev/null
+       then
+           sudo iptables -I FORWARD -i ppp+ -j VPN_FORWARD
+           sudo iptables -I FORWARD -o ppp+ -j VPN_FORWARD
+           sudo iptables -A VPN_FORWARD -j DROP
+       fi
+       sudo iptables $op VPN_FORWARD -i ppp+ -o ppp+ -j RETURN
+       sudo iptables $op VPN_FORWARD -i ppp+ -d $cidr -j RETURN
+       sudo iptables $op VPN_FORWARD -s $cidr -o ppp+ -j RETURN
+   else
+       sudo iptables $op FORWARD -i ppp+ -d $cidr -j ACCEPT
+       sudo iptables $op FORWARD -s $cidr -o ppp+ -j ACCEPT
+       sudo iptables $op FORWARD -i ppp+ -o ppp+ -j ACCEPT
+   fi
    sudo iptables $op INPUT -i ppp+ -m udp -p udp --dport 53 -j ACCEPT
    sudo iptables $op INPUT -i ppp+ -m tcp -p tcp --dport 53 -j ACCEPT
    sudo iptables -t nat $op PREROUTING -i ppp+ -p tcp -m tcp --dport 53 -j  DNAT --to-destination $local_ip
    sudo iptables -t nat $op PREROUTING -i ppp+ -p udp -m udp --dport 53 -j  DNAT --to-destination $local_ip
 
-   if grep "vpcrouter" /var/cache/cloud/cmdline &> /dev/null
+   if $is_vpc
    then
        return
    fi

test/integration/component/test_vpc_network.py

@@ -2332,11 +2332,6 @@ class TestVPCNetworkGc(cloudstackTestCase):
         wait_for_cleanup(self.apiclient,
                          ["network.gc.interval", "network.gc.wait"])
 
-        #Bug???: Network Acls are not cleared
-        netacls = NetworkACL.list(self.apiclient, networkid=self.network_1.id)
-        self.debug("List of NetACLS %s" % netacls)
-        self.assertEqual(netacls, None, "Netacls were not cleared after network GC thread is run")
-
         lbrules = LoadBalancerRule.list(self.apiclient, networkid=self.network_1.id)
         self.debug("List of LB Rules %s" % lbrules)
         self.assertEqual(lbrules, None, "LBrules were not cleared after network GC thread is run")

tools/marvin/marvin/configGenerator.py

@@ -93,11 +93,6 @@ class trafficType(object):
                 if 'vmware' in labeldict.keys() else None
             self.simulator = labeldict['simulator']\
                 if 'simulator' in labeldict.keys() else None
-        #{
-        #    'xen' : 'cloud-xen',
-        #    'kvm' : 'cloud-kvm',
-        #    'vmware' : 'cloud-vmware'
-        #}
 
 
 class pod(object):

tools/marvin/marvin/deployAndRun.py

@@ -15,82 +15,153 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import deployDataCenter
-import TestCaseExecuteEngine
+from tcExecuteEngine import TestCaseExecuteEngine
 import sys
+import os
+import traceback
+import time
 from argparse import ArgumentParser
-if __name__ == "__main__":
+from marvinInit import MarvinInit
+from marvin.codes import (SUCCESS,
+                          FAILED,
+                          EXCEPTION,
+                          UNKNOWN_ERROR
+                          )
 
+
+parser = None
+
+
+def printAndExit():
+    '''
+    Prints pretty message for parser and exit
+    '''
+    global parser
+    if parser is not None:
+        parser.print_usage()
+        exit(1)
+
+
+def parseAndCheck():
+    '''
+    Parses,reads the options and verifies for the config file
+    '''
+    global parser
     parser = ArgumentParser()
 
-    parser.add_argument("-d", "--directory", dest="testCaseFolder",
-                        help="the test case directory")
-    parser.add_argument("-f", "--file", dest="module",
-                        help="run tests in the given file")
-    parser.add_argument("-r", "--result", dest="result",
-                        help="test result log file", default='/tmp/t.log')
-    parser.add_argument("-t", "--client", dest="testcaselog",
-                        help="test case log file", default='/tmp/r.log')
+    parser.add_argument("-d", "--tcpath", dest="tcpath",
+                        help="the test case directory or file path")
     parser.add_argument("-c", "--config", action="store",
                         default="./datacenterCfg", dest="config",
                         help="the path where the json config file generated,\
- by default is ./datacenterCfg")
+                        by default is ./datacenterCfg")
     parser.add_argument("-l", "--load", dest="load", action="store_true",
                         help="only load config, do not deploy,\
- it will only run testcase")
+                        it will only run testcase")
     parser.add_argument("-n", "--num", dest="number",
-                        help="how many times you want run the test case")
+                        help="how many times you want to run the tests")
 
     options = parser.parse_args()
+    cfg_file = options.config
+    tc_path = options.tcpath
+    load_flag = options.load
+    num_iter = 1 if options.number is None else int(options.number)
 
-    testResultLogFile = None
-    if options.result is not None:
-        testResultLogFile = options.result
+    '''
+    Check if the config file is None or not and exit accordingly
+    '''
+    if cfg_file is None:
+        printAndExit()
+    return {"cfg_file": cfg_file,
+            "load_flag": load_flag,
+            "tc_path": tc_path,
+            "num_iter": num_iter}
 
-    testCaseLogFile = None
-    if options.testcaselog is not None:
-        testCaseLogFile = options.testcaselog
-    deploy = deployDataCenter.deployDataCenters(options.config)
-    if options.load:
-        deploy.loadCfg()
-    else:
-        deploy.deploy()
-    iterates = 1
-    if options.number is not None:
-        if options.number == "loop":
-            iterates = sys.maxint
+
+def startMarvin(cfg_file, load_flag):
+    '''
+    Initialize the Marvin
+    '''
+    try:
+        obj_marvininit = MarvinInit(cfg_file, load_flag)
+        if obj_marvininit.init() == SUCCESS:
+            testClient = obj_marvininit.getTestClient()
+            tcRunLogger = obj_marvininit.getLogger()
+            parsedConfig = obj_marvininit.getParsedConfig()
+            debugStream = obj_marvininit.getDebugFile()
+            return {"tc_client": testClient,
+                    "tc_runlogger": tcRunLogger,
+                    "tc_parsedcfg": parsedConfig,
+                    "tc_debugstream": debugStream}
         else:
-            try:
-                iterates = int(options.number)
-            except:
-                iterates = 1
-
-    if options.testCaseFolder is None:
-        if options.module is None:
-            parser.print_usage()
+            print "\nMarvin Initialization Failed"
             exit(1)
+    except Exception, e:
+            print "\n Exception occurred while starting Marvin %s" % str(e)
+            exit(1)
+
+
+def runTCs(num_iter, inp1, inp2):
+    '''
+    Run Test Cases based upon number of iterations
+    '''
+    n = 0
+    while(n < num_iter):
+        engine = TestCaseExecuteEngine(inp2["tc_client"],
+                                       inp2["tc_parsedcfg"],
+                                       inp2["tc_runlogger"],
+                                       inp2["tc_debugstream"])
+        if inp1["tc_file"] is not None:
+            engine.loadTestsFromFile(inp1["tc_file"])
         else:
-            n = 0
-            while(n < iterates):
-                engine = \
-                    TestCaseExecuteEngine.TestCaseExecuteEngine(
-                        deploy.testClient,
-                        deploy.getCfg(
-                        ),
-                        testCaseLogFile,
-                        testResultLogFile)
-                engine.loadTestsFromFile(options.module)
-                engine.run()
-                n = n + 1
+            engine.loadTestsFromDir(inp1["tc_dir"])
+        engine.run()
+        n = n + 1
+
+
+def checkTCPath(tc_path):
+    '''
+    Verifies if the tc_path is a folder or file and its existence
+    '''
+    ret = {"tc_file": None, "tc_dir": None}
+    check = True
+    if tc_path is None:
+        printAndExit()
     else:
-        n = 0
-        while(n < iterates):
-            engine = TestCaseExecuteEngine.TestCaseExecuteEngine(
-                deploy.testClient,
-                deploy.getCfg(
-                ),
-                testCaseLogFile,
-                testResultLogFile)
-            engine.loadTestsFromDir(options.testCaseFolder)
-            engine.run()
-            n = n + 1
+        if os.path.isfile(tc_path):
+            ret["tc_file"] = tc_path
+        elif os.path.isdir(tc_path):
+            ret["tc_dir"] = tc_path
+        else:
+            check = False
+    if check is False:
+        print"\nTC Path is Invalid.So Exiting"
+        exit(1)
+
+    return ret
+
+if __name__ == "__main__":
+
+    '''
+    1. Parse and Check
+    '''
+    out1 = parseAndCheck()
+    print "\nStep1 :Parsing Options And Check Went Fine"
+
+    '''
+    2. Start Marvin
+    '''
+    out2 = startMarvin(out1["cfg_file"], out1["load_flag"])
+    print "\nStep2: Marvin Initialization Went Fine"
+
+    '''
+    3. Check TC folder or Module and Path existence
+    '''
+    out3 = checkTCPath(out1["tc_path"])
+    print "\nStep3: TC Path Check Went Fine"
+
+    '''
+    4. Run TCs
+    '''
+    runTCs(out1["num_iter"], out3, out2)
+    print "\nStep4: TC Running Finished"

tools/marvin/marvin/integration/lib/base.py

@@ -246,13 +246,17 @@ class VirtualMachine:
         if zone.securitygroupsenabled:
             list_security_groups = SecurityGroup.list(
                 apiclient,
+                account=cmd.account,
+                domainid=cmd.domainid,
                 securitygroupname="basic_sec_grp"
             )
 
             if not isinstance(list_security_groups, list):
                 basic_mode_security_group = SecurityGroup.create(
                     apiclient,
-                    {"name": "basic_sec_grp"}
+                    {"name": "basic_sec_grp"},
+                    cmd.account,
+                    cmd.domainid,
                 )
                 sec_grp_services = {
                     "protocol": "TCP",

tools/marvin/marvin/marvinInit.py

@@ -39,14 +39,16 @@ import sys
 import time
 import os
 import logging
+import string
+import random
 
 
 class MarvinInit:
-    def __init__(self, config_file, load_flag):
+    def __init__(self, config_file, load_flag, log_folder_path=None):
         self.__configFile = config_file
         self.__loadFlag = load_flag
         self.__parsedConfig = None
-        self.__logFolderPath = None
+        self.__logFolderPath = log_folder_path
         self.__tcRunLogger = None
         self.__testClient = None
         self.__tcRunDebugFile = None
@@ -111,15 +113,21 @@ class MarvinInit:
                      for a given test run are available under a given
                      timestamped folder
             '''
-            log_config = self.__parsedConfig.logger
             temp_path = "".join(str(time.time()).split("."))
-            if log_config is not None:
-                if log_config.LogFolderPath is not None:
-                    self.logFolderPath = log_config.LogFolderPath + temp_path
+            if self.__logFolderPath is None:
+                log_config = self.__parsedConfig.logger
+                if log_config is not None:
+                    if log_config.LogFolderPath is not None:
+                        self.logFolderPath = log_config.LogFolderPath + '/' + temp_path
+                    else:
+                        self.logFolderPath = temp_path
                 else:
                     self.logFolderPath = temp_path
             else:
-                self.logFolderPath = temp_path
+                self.logFolderPath = self.__logFolderPath + '/' + temp_path
+            if os.path.exists(self.logFolderPath):
+                self.logFolderPath = self.logFolderPath \
+                                     + ''.join(random.choice(string.ascii_uppercase + string.digits) for x in range(3))
             os.makedirs(self.logFolderPath)
             '''
             Log File Paths

tools/marvin/marvin/marvinPlugin.py

@@ -68,6 +68,7 @@ class MarvinPlugin(Plugin):
                 self.enabled = True
         self.configFile = options.config_file
         self.loadFlag = options.load
+        self.logFolderPath = options.log_folder_path
         self.conf = conf
         '''
         Initializes the marvin with required settings
@@ -87,6 +88,9 @@ class MarvinPlugin(Plugin):
         parser.add_option("--load", action="store_true", default=False,
                           dest="load",
                           help="Only load the deployment configuration given")
+        parser.add_option("--log-folder-path", action="store", default=None,
+                          dest="log_folder_path",
+                          help="Path to the folder where log files will be stored")
         Plugin.options(self, parser, env)
 
     def wantClass(self, cls):
@@ -165,7 +169,7 @@ class MarvinPlugin(Plugin):
         Creates a debugstream for tc debug log
         '''
         try:
-            obj_marvininit = MarvinInit(self.configFile, self.loadFlag)
+            obj_marvininit = MarvinInit(self.configFile, self.loadFlag, self.logFolderPath)
             if obj_marvininit.init() == SUCCESS:
                 self.testClient = obj_marvininit.getTestClient()
                 self.tcRunLogger = obj_marvininit.getLogger()

tools/marvin/marvin/tcExecuteEngine.py

@@ -22,14 +22,8 @@ import logging
 from functools import partial
 
 
-def testCaseLogger(message, logger=None):
-    if logger is not None:
-        logger.debug(message)
-
-
 class TestCaseExecuteEngine(object):
-    def __init__(self, testclient, config, testcaseLogFile=None,
-                 testResultLogFile=None):
+    def __init__(self, testclient, config, tc_logger=None, debug_stream=None):
         """
         Initialize the testcase execution engine, just the basics here
         @var testcaseLogFile: client log file
@@ -37,29 +31,11 @@ class TestCaseExecuteEngine(object):
         """
         self.testclient = testclient
         self.config = config
-        self.logformat =\
-            logging.Formatter(
-                "%(asctime)s - %(levelname)s - %(name)s - %(message)s")
+        self.tcRunLogger = tc_logger
+        self.debugStream = debug_stream
         self.loader = unittest.loader.TestLoader()
         self.suite = None
 
-        if testcaseLogFile is not None:
-            self.logfile = testcaseLogFile
-            self.logger = logging.getLogger("TestCaseExecuteEngine")
-            fh = logging.FileHandler(self.logfile)
-            fh.setFormatter(self.logformat)
-            self.logger.addHandler(fh)
-            self.logger.setLevel(logging.DEBUG)
-        if testResultLogFile is not None:
-            ch = logging.StreamHandler()
-            ch.setLevel(logging.ERROR)
-            ch.setFormatter(self.logformat)
-            self.logger.addHandler(ch)
-            fp = open(testResultLogFile, "w")
-            self.testResultLogFile = fp
-        else:
-            self.testResultLogFile = sys.stdout
-
     def loadTestsFromDir(self, testDirectory):
         """ Load the test suites from a package with multiple test files """
         self.suite = self.loader.discover(testDirectory)
@@ -77,18 +53,11 @@ class TestCaseExecuteEngine(object):
             if isinstance(test, unittest.BaseTestSuite):
                 self.injectTestCase(test)
             else:
-                #logger bears the name of the test class
-                testcaselogger = logging.getLogger("%s" % (test))
-                fh = logging.FileHandler(self.logfile)
-                fh.setFormatter(self.logformat)
-                testcaselogger.addHandler(fh)
-                testcaselogger.setLevel(logging.DEBUG)
-
                 #inject testclient and logger into each unittest
+                self.tcRunLogger.name = test.__str__()
                 setattr(test, "testClient", self.testclient)
                 setattr(test, "config", self.config)
-                setattr(test, "debug", partial(testCaseLogger,
-                                               logger=testcaselogger))
+                setattr(test, "debug", self.tcRunLogger.debug)
                 setattr(test.__class__, "clstestclient", self.testclient)
                 if hasattr(test, "user"):
                     # attribute when test is entirely executed as user
@@ -98,5 +67,5 @@ class TestCaseExecuteEngine(object):
 
     def run(self):
         if self.suite:
-            unittest.TextTestRunner(stream=self.testResultLogFile,
+            unittest.TextTestRunner(stream=self.debugStream,
                                     verbosity=2).run(self.suite)

ui/scripts/cloudStack.js

@@ -168,8 +168,8 @@
                             expires: 1
                         });
 
-                        g_KVMsnapshotenabled = json.listcapabilitiesresponse.capability.KVMsnapshotenabled; //boolean
-                        $.cookie('KVMsnapshotenabled', g_KVMsnapshotenabled, {
+                        g_kvmsnapshotenabled = json.listcapabilitiesresponse.capability.kvmsnapshotenabled; //boolean
+                        $.cookie('kvmsnapshotenabled', g_kvmsnapshotenabled, {
                             expires: 1
                         });                        
                                                
@@ -309,8 +309,8 @@
                                     expires: 1
                                 });
 
-                                g_KVMsnapshotenabled = json.listcapabilitiesresponse.capability.KVMsnapshotenabled; //boolean
-                                $.cookie('KVMsnapshotenabled', g_KVMsnapshotenabled, {
+                                g_kvmsnapshotenabled = json.listcapabilitiesresponse.capability.kvmsnapshotenabled; //boolean
+                                $.cookie('kvmsnapshotenabled', g_kvmsnapshotenabled, {
                                     expires: 1
                                 });   
                                 
@@ -386,7 +386,7 @@
                         g_timezoneoffset = null;
                         g_timezone = null;
                         g_supportELB = null;
-                        g_KVMsnapshotenabled = null;
+                        g_kvmsnapshotenabled = null;
                         g_regionsecondaryenabled = null;
                         g_loginCmdText = null;
 

ui/scripts/instances.js

@@ -2184,7 +2184,7 @@
         } else if (jsonObj.state == 'Running') {
             allowedActions.push("stop");
             allowedActions.push("restart");
-            if (jsonObj.hypervisor != 'KVM' || g_KVMsnapshotenabled == true)
+            if (jsonObj.hypervisor != 'KVM' || g_kvmsnapshotenabled == true)
                 allowedActions.push("snapshot");
             allowedActions.push("destroy");            
             allowedActions.push("reset");
@@ -2214,7 +2214,7 @@
             allowedActions.push("start");
             allowedActions.push("destroy");
             allowedActions.push("reset");
-            if (jsonObj.hypervisor != 'KVM' || g_KVMsnapshotenabled == true)
+            if (jsonObj.hypervisor != 'KVM' || g_kvmsnapshotenabled == true)
                 allowedActions.push("snapshot");
             allowedActions.push("scaleUp");  //when vm is stopped, scaleUp is supported for all hypervisors 
             allowedActions.push("changeAffinity");

ui/scripts/sharedFunctions.js

@@ -25,7 +25,7 @@ var g_enableLogging = false;
 var g_timezoneoffset = null;
 var g_timezone = null;
 var g_supportELB = null;
-var g_KVMsnapshotenabled =  null;
+var g_kvmsnapshotenabled =  null;
 var g_regionsecondaryenabled = null;
 var g_userPublicTemplateEnabled = "true";
 var g_cloudstackversion = null;

ui/scripts/storage.js

@@ -70,6 +70,10 @@
                         add: {
                             label: 'label.add.volume',
 
+                            preFilter: function(args) {
+                                return !args.context.instances;
+                            },
+
                             messages: {
                                 confirm: function(args) {
                                     return 'message.add.volume';
@@ -252,6 +256,9 @@
                         uploadVolume: {
                             isHeader: true,
                             label: 'label.upload.volume',
+                            preFilter: function(args) {
+                                return !args.context.instances;
+                            },
                             messages: {
                                 notification: function() {
                                     return 'label.upload.volume';
@@ -1920,7 +1927,7 @@
         if (jsonObj.hypervisor != "Ovm" && jsonObj.state == "Ready") {        	
         	if (jsonObj.hypervisor == 'KVM') { 
         		if (jsonObj.vmstate == 'Running') {        			
-        			if (g_KVMsnapshotenabled == true) { //"kvm.snapshot.enabled" flag should be taken to account only when snapshot is being created for Running vm (CLOUDSTACK-4428)
+        			if (g_kvmsnapshotenabled == true) { //"kvm.snapshot.enabled" flag should be taken to account only when snapshot is being created for Running vm (CLOUDSTACK-4428)
             			allowedActions.push("takeSnapshot");
         	            allowedActions.push("recurringSnapshot");
             		}         			

ui/scripts/system.js

@@ -8805,19 +8805,21 @@
                                         var zoneObjs = json.listzonesresponse.zone;                                        
                                         if (zoneObjs != null) {
                                         	for (var i = 0; i < zoneObjs.length; i++) {
+                                        		var currentPage = 1;
                                         		$.ajax({
     	                                            url: createURL('listRouters'),
     	                                            data: {
-    	                                            	zoneid: zoneObjs[i].id
+    	                                            	zoneid: zoneObjs[i].id,
+    	                                            	listAll: true,
+                                            			page: currentPage,
+                                            	        pagesize: pageSize //global variable
     	                                            },
     	                                            async: false,
     	                                            success: function(json) {
     	                                            	if (json.listroutersresponse.count != undefined) {
-    	                                            		zoneObjs[i].routerCount = json.listroutersresponse.count;    
-    	                                            		    	                                            		
-    	                                            		var routerCountFromAllPages = zoneObjs[i].routerCount;                                                	
-    	                                                	var currentPage = 1;
-    	                                                	var routerCountFromFirstPageToCurrentPage = 0;  
+    	                                            		zoneObjs[i].routerCount = json.listroutersresponse.count;        	                                            		    	                                            		
+    	                                            		var routerCountFromAllPages = json.listroutersresponse.count;        
+    	                                                	var routerCountFromFirstPageToCurrentPage = json.listroutersresponse.router.length;  
     	                                                	var routerRequiresUpgrade = 0;    	                                                	                                                	
     	                                                	var callListApiWithPage = function() {                                                		
     	                                                		$.ajax({
@@ -8825,6 +8827,7 @@
     	                                                    		async: false,
     	                                                    		data: {    	                                                    			
     	                                                    			zoneid: zoneObjs[i].id,
+    	                                                    			listAll: true,
     	                                                    			page: currentPage,
     	                                                    	        pagesize: pageSize //global variable
     	                                                    		},
@@ -8843,7 +8846,10 @@
     	                                                    		}
     	                                        				});                                                		
     	                                                	}    	                                                	
-    	                                                	callListApiWithPage();                  	
+    	                                                	if (routerCountFromFirstPageToCurrentPage < routerCountFromAllPages) {
+                                                				currentPage++;
+                                                				callListApiWithPage();
+                                                			}              	
     	                                                	zoneObjs[i].routerRequiresUpgrade = routerRequiresUpgrade;
     	                                            		
     	                                            	} else {
@@ -9024,26 +9030,29 @@
                                         var podObjs = json.listpodsresponse.pod;
                                         if (podObjs != null) {
                                             for (var i = 0; i < podObjs.length; i++) {
-                                                $.ajax({
+                                            	var currentPage = 1;
+                                            	$.ajax({
                                                     url: createURL('listRouters'),
                                                     data: {
-                                                        podid: podObjs[i].id
+                                                        podid: podObjs[i].id,
+                                                        listAll: true,
+                                                        page: currentPage,
+                                                        pagesize: pageSize //global variable
                                                     },
                                                     async: false,
                                                     success: function (json) {
                                                         if (json.listroutersresponse.count != undefined) {
                                                             podObjs[i].routerCount = json.listroutersresponse.count;
-
-                                                            var routerCountFromAllPages = podObjs[i].routerCount;
-                                                            var currentPage = 1;
-                                                            var routerCountFromFirstPageToCurrentPage = 0;
-                                                            var routerRequiresUpgrade = 0;
+                                                            var routerCountFromAllPages = json.listroutersresponse.count;                                                            
+                                                            var routerCountFromFirstPageToCurrentPage = json.listroutersresponse.router.length;
+                                                            var routerRequiresUpgrade = 0;                                                            
                                                             var callListApiWithPage = function () {
                                                                 $.ajax({
                                                                     url: createURL('listRouters'),
                                                                     async: false,
                                                                     data: {
                                                                         podid: podObjs[i].id,
+                                                                        listAll: true,
                                                                         page: currentPage,
                                                                         pagesize: pageSize //global variable
                                                                     },
@@ -9061,8 +9070,11 @@
                                                                         }
                                                                     }
                                                                 });
-                                                            }
-                                                            callListApiWithPage();
+                                                            }                                                                
+                                                            if (routerCountFromFirstPageToCurrentPage < routerCountFromAllPages) {
+                                                                currentPage++;
+                                                                callListApiWithPage();
+                                                            }                                                            
                                                             podObjs[i].routerRequiresUpgrade = routerRequiresUpgrade;
 
                                                         } else {
@@ -9245,19 +9257,21 @@
                                         var clusterObjs = json.listclustersresponse.cluster;
                                         if (clusterObjs != null) {
                                             for (var i = 0; i < clusterObjs.length; i++) {
-                                                $.ajax({
+                                            	var currentPage = 1;
+                                            	$.ajax({
                                                     url: createURL('listRouters'),
                                                     data: {
-                                                        clusterid: clusterObjs[i].id
+                                                        clusterid: clusterObjs[i].id,
+                                                        listAll: true,
+                                                        page: currentPage,
+                                                        pagesize: pageSize //global variable
                                                     },
                                                     async: false,
                                                     success: function (json) {
                                                         if (json.listroutersresponse.count != undefined) {
                                                             clusterObjs[i].routerCount = json.listroutersresponse.count;
-
-                                                            var routerCountFromAllPages = clusterObjs[i].routerCount;
-                                                            var currentPage = 1;
-                                                            var routerCountFromFirstPageToCurrentPage = 0;
+                                                            var routerCountFromAllPages = json.listroutersresponse.count;                                                            
+                                                            var routerCountFromFirstPageToCurrentPage = json.listroutersresponse.router.length;
                                                             var routerRequiresUpgrade = 0;
                                                             var callListApiWithPage = function () {
                                                                 $.ajax({
@@ -9265,6 +9279,7 @@
                                                                     async: false,
                                                                     data: {
                                                                         clusterid: clusterObjs[i].id,
+                                                                        listAll: true,
                                                                         page: currentPage,
                                                                         pagesize: pageSize //global variable
                                                                     },
@@ -9283,7 +9298,10 @@
                                                                     }
                                                                 });
                                                             }
-                                                            callListApiWithPage();
+                                                            if (routerCountFromFirstPageToCurrentPage < routerCountFromAllPages) {
+                                                                currentPage++;
+                                                                callListApiWithPage();
+                                                            }
                                                             clusterObjs[i].routerRequiresUpgrade = routerRequiresUpgrade;
 
                                                         } else {
@@ -14787,7 +14805,7 @@
                                                 var items = [];
                                                 items.push({
                                                     id: "SMB",
-                                                    description: "SMB/cifs"
+                                                    description: "SMB/CIFS"
                                                 });
                                                 args.response.success({
                                                     data: items
@@ -16255,7 +16273,7 @@
                                                 	    description: 'NFS'
                                                 	}, {
                                                 	    id: 'SMB',
-                                                	    description: 'SMB/cifs'
+                                                	    description: 'SMB/CIFS'
                                                 	}, {
                                                 	    id: 'S3',
                                                 	    description: 'S3'
@@ -16356,7 +16374,7 @@
                                                             $form.find('.form-item[rel=sockettimeout]').css('display', 'inline-block');
 
                                                             $form.find('.form-item[rel=createNfsCache]').find('input').attr('checked', 'checked');
-                                                            $form.find('.form-item[rel=createNfsCache]').find('input').attr('disabled', 'disabled');  //Create NFS staging is required for S3 at this moment. So, disallow user to uncheck "Create NFS Secondary Staging" checkbox
+                                                            //$form.find('.form-item[rel=createNfsCache]').find('input').attr('disabled', 'disabled');  //This checkbox should not be disabled any more because NFS staging (of a zone) might already exist (from "NFS secondary storage => Prepare Object Store Migration => NFS staging")
                                                             $form.find('.form-item[rel=createNfsCache]').css('display', 'inline-block');
                                                             $form.find('.form-item[rel=nfsCacheZoneid]').css('display', 'inline-block');
                                                             $form.find('.form-item[rel=nfsCacheNfsServer]').css('display', 'inline-block');

ui/scripts/ui/widgets/listView.js

@@ -1685,7 +1685,7 @@
             $.each(listViewData.actions, function(actionName, action) {
                 if (!action.isHeader || (
                     action.preFilter && !action.preFilter({
-                        context: listViewData.context ? listViewData.context : cloudStack.context
+                        context: $listView.data('view-args').context ? $listView.data('view-args').context : cloudStack.context
                     })
                 )) return true;
 

ui/scripts/zoneWizard.js

@@ -1523,7 +1523,7 @@
                             	var items = []; 
                             	items.push({
                                     id: "SMB",
-                                    description: "SMB/cifs"
+                                    description: "SMB/CIFS"
                                 });
                             	args.response.success({
                                     data: items
@@ -1845,7 +1845,7 @@
                                     	(2) Provider "SMB" which is handled by UI is not returned from "listStorageProviders&type=image" 
                                     	*/
                         				storageproviders.push({ id: 'NFS', description: 'NFS'});
-                        				storageproviders.push({ id: 'SMB', description: 'SMB/cifs'});
+                        				storageproviders.push({ id: 'SMB', description: 'SMB/CIFS'});
                         				storageproviders.push({ id: 'S3', description: 'S3'});
                         				storageproviders.push({ id: 'Swift', description: 'Swift'});                        				
                         			}