bug 9842: Added task to update user stats agg bytes in sync with aggregation range

status 9842: resolved fixed
This commit is contained in:
kishan 2011-10-31 17:12:12 +05:30
parent f34ec52031
commit 284ee3909e
14 changed files with 277 additions and 99 deletions

View File

@ -62,7 +62,13 @@ public class UserStatisticsVO {
private long currentBytesReceived;
@Column(name="current_bytes_sent")
private long currentBytesSent;
private long currentBytesSent;
@Column(name="agg_bytes_received")
private long aggBytesReceived;
@Column(name="agg_bytes_sent")
private long aggBytesSent;
protected UserStatisticsVO() {
}
@ -77,7 +83,7 @@ public class UserStatisticsVO {
this.netBytesReceived = 0;
this.netBytesSent = 0;
this.currentBytesReceived = 0;
this.currentBytesSent = 0;
this.currentBytesSent = 0;
}
public long getAccountId() {
@ -139,5 +145,21 @@ public class UserStatisticsVO {
public void setNetBytesSent(long netBytesSent) {
this.netBytesSent = netBytesSent;
}
public long getAggBytesReceived() {
return aggBytesReceived;
}
public void setAggBytesReceived(long aggBytesReceived) {
this.aggBytesReceived = aggBytesReceived;
}
public long getAggBytesSent() {
return aggBytesSent;
}
public void setAggBytesSent(long aggBytesSent) {
this.aggBytesSent = aggBytesSent;
}
}

View File

@ -18,13 +18,16 @@
package com.cloud.network.router;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@ -337,10 +340,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
private boolean _disable_rp_filter = false;
int _routerExtraPublicNics = 2;
private int _usageAggregationRange = 1440;
private String _usageTimeZone = "GMT";
private long mgmtSrvrId = MacAddress.getMacAddress().toLong();
ScheduledExecutorService _executor;
ScheduledExecutorService _checkExecutor;
ScheduledExecutorService _networkStatsUpdateExecutor;
Account _systemAcct;
@ -592,6 +598,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
_executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor"));
_checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor"));
_networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater"));
final ComponentLocator locator = ComponentLocator.getCurrentLocator();
@ -645,6 +652,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
_systemAcct = _accountService.getSystemAccount();
String aggregationRange = configs.get("usage.stats.job.aggregation.range");
_usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440);
_usageTimeZone = configs.get("usage.aggregation.timezone");
if(_usageTimeZone == null){
_usageTimeZone = "GMT";
}
_agentMgr.registerForHostEvents(this, true, false, false);
s_logger.info("DomainRouterManager is configured.");
@ -661,10 +675,41 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
public boolean start() {
if (_routerStatsInterval > 0){
_executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS);
//Schedule Network stats update task
TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone);
Calendar cal = Calendar.getInstance(usageTimezone);
cal.setTime(new Date());
long endDate = 0;
int HOURLY_TIME = 60;
final int DAILY_TIME = 60 * 24;
if (_usageAggregationRange == DAILY_TIME) {
cal.roll(Calendar.DAY_OF_YEAR, false);
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.roll(Calendar.DAY_OF_YEAR, true);
cal.add(Calendar.MILLISECOND, -1);
endDate = cal.getTime().getTime();
} else if (_usageAggregationRange == HOURLY_TIME) {
cal.roll(Calendar.HOUR_OF_DAY, false);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
cal.roll(Calendar.HOUR_OF_DAY, true);
cal.add(Calendar.MILLISECOND, -1);
endDate = cal.getTime().getTime();
} else {
endDate = cal.getTime().getTime();
}
_networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS);
}else{
s_logger.debug("router.stats.interval - " + _routerStatsInterval+ " so not scheduling the router stats thread");
}
_checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _checkRouterInterval, _checkRouterInterval, TimeUnit.SECONDS);
_checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _checkRouterInterval, _checkRouterInterval, TimeUnit.SECONDS);
return true;
}
@ -791,6 +836,27 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian
}
}
protected class NetworkStatsUpdateTask implements Runnable {
public NetworkStatsUpdateTask() {
}
@Override
public void run() {
try{
if(_statsDao.updateAggStats()){
s_logger.debug("Succussfully updated aggregate network stats");
} else {
s_logger.debug("Failed to update aggregate network stats");
}
} catch (Exception e){
s_logger.debug("Failed to update aggregate network stats", e);
}
}
}
protected void updateRoutersRedundantState(List<DomainRouterVO> routers) {
boolean updated = false;
for (DomainRouterVO router : routers) {

View File

@ -27,6 +27,7 @@ import com.cloud.upgrade.dao.Upgrade218to22Premium;
import com.cloud.upgrade.dao.Upgrade2210to2211;
import com.cloud.upgrade.dao.Upgrade2211to2212Premium;
import com.cloud.upgrade.dao.Upgrade2212to2213;
import com.cloud.upgrade.dao.Upgrade2213to30Premium;
import com.cloud.upgrade.dao.Upgrade221to222Premium;
import com.cloud.upgrade.dao.Upgrade222to224Premium;
import com.cloud.upgrade.dao.Upgrade224to225;
@ -34,7 +35,6 @@ import com.cloud.upgrade.dao.Upgrade225to226;
import com.cloud.upgrade.dao.Upgrade227to228Premium;
import com.cloud.upgrade.dao.Upgrade228to229;
import com.cloud.upgrade.dao.Upgrade229to2210;
import com.cloud.upgrade.dao.Upgrade2213to30;
import com.cloud.upgrade.dao.UpgradeSnapshot217to224;
import com.cloud.upgrade.dao.UpgradeSnapshot223to224;
import com.cloud.upgrade.dao.VersionDaoImpl;
@ -46,23 +46,23 @@ public class PremiumDatabaseUpgradeChecker extends DatabaseUpgradeChecker {
public PremiumDatabaseUpgradeChecker() {
_dao = ComponentLocator.inject(VersionDaoImpl.class);
_upgradeMap.put("2.1.7", new DbUpgrade[] { new Upgrade217to218(), new Upgrade218to22Premium(), new Upgrade221to222Premium(), new UpgradeSnapshot217to224(), new Upgrade222to224Premium(),
new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.1.8", new DbUpgrade[] { new Upgrade218to22Premium(), new Upgrade221to222Premium(), new UpgradeSnapshot217to224(), new Upgrade222to224Premium(),
new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213() , new Upgrade2213to30()});
new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213() , new Upgrade2213to30Premium()});
_upgradeMap.put("2.1.9", new DbUpgrade[] { new Upgrade218to22Premium(), new Upgrade221to222Premium(), new UpgradeSnapshot217to224(), new Upgrade222to224Premium(),
new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.1", new DbUpgrade[] { new Upgrade221to222Premium(), new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.2", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.3", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.4", new DbUpgrade[] { new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.5", new DbUpgrade[] { new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.6", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.7", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.8", new DbUpgrade[] { new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.9", new DbUpgrade[] { new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.10", new DbUpgrade[] { new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.11", new DbUpgrade[] { new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.12", new DbUpgrade[] { new Upgrade2212to2213(), new Upgrade2213to30()});
_upgradeMap.put("2.2.13", new DbUpgrade[] { new Upgrade2213to30()});
new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.1", new DbUpgrade[] { new Upgrade221to222Premium(), new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.2", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.3", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.4", new DbUpgrade[] { new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.5", new DbUpgrade[] { new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.6", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.7", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.8", new DbUpgrade[] { new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.9", new DbUpgrade[] { new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.10", new DbUpgrade[] { new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.11", new DbUpgrade[] { new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.12", new DbUpgrade[] { new Upgrade2212to2213(), new Upgrade2213to30Premium()});
_upgradeMap.put("2.2.13", new DbUpgrade[] { new Upgrade2213to30Premium()});
}
}

View File

@ -0,0 +1,65 @@
/**
* Copyright (C) 2011 Citrix Systems, Inc. All rights reserved
*
* This software is licensed under the GNU General Public License v3 or later.
*
* It is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or any later version.
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
package com.cloud.upgrade.dao;
import java.io.File;
import java.sql.Connection;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.Script;
public class Upgrade2213to30Premium implements DbUpgrade {
final static Logger s_logger = Logger.getLogger(Upgrade2213to30Premium.class);
@Override
public String[] getUpgradableVersionRange() {
return new String[] { "2.2.13", "3.0.0"};
}
@Override
public String getUpgradedVersion() {
return "3.0.0";
}
@Override
public boolean supportsRollingUpgrade() {
return true;
}
@Override
public File[] getPrepareScripts() {
String script = Script.findScript("", "db/db/schema-2213to30-premium.sql");
if (script == null) {
throw new CloudRuntimeException("Unable to find db/schema-2213to30-premium.sql");
}
return new File[] { new File(script) };
}
@Override
public void performDataMigration(Connection conn) {
}
@Override
public File[] getCleanupScripts() {
return null;
}
}

View File

@ -50,26 +50,19 @@ public class UsageNetworkVO {
@Column(name="bytes_received")
private long bytesReceived;
@Column(name="net_bytes_received")
private long netBytesReceived;
@Column(name="agg_bytes_received")
private long aggBytesReceived;
@Column(name="net_bytes_sent")
private long netBytesSent;
@Column(name="agg_bytes_sent")
private long aggBytesSent;
@Column(name="current_bytes_received")
private long currentBytesReceived;
@Column(name="current_bytes_sent")
private long currentBytesSent;
@Column(name="event_time_millis")
private long eventTimeMillis = 0;
protected UsageNetworkVO() {
}
public UsageNetworkVO(Long accountId, long zoneId, long hostId, String hostType, Long networkId, long bytesSent, long bytesReceived, long netBytesReceived, long netBytesSent, long currentBytesReceived,
long currentBytesSent, long eventTimeMillis) {
public UsageNetworkVO(Long accountId, long zoneId, long hostId, String hostType, Long networkId, long bytesSent, long bytesReceived, long aggBytesReceived, long aggBytesSent, long eventTimeMillis) {
this.accountId = accountId;
this.zoneId = zoneId;
this.hostId = hostId;
@ -77,10 +70,8 @@ public class UsageNetworkVO {
this.networkId = networkId;
this.bytesSent = bytesSent;
this.bytesReceived = bytesReceived;
this.netBytesReceived = netBytesReceived;
this.netBytesSent = netBytesSent;
this.currentBytesReceived = currentBytesReceived;
this.currentBytesSent = currentBytesSent;
this.aggBytesReceived = aggBytesReceived;
this.aggBytesSent = aggBytesSent;
this.eventTimeMillis = eventTimeMillis;
}
@ -115,22 +106,6 @@ public class UsageNetworkVO {
this.bytesReceived = bytesReceived;
}
public long getCurrentBytesReceived() {
return currentBytesReceived;
}
public long getCurrentBytesSent() {
return currentBytesSent;
}
public long getNetBytesReceived() {
return netBytesReceived;
}
public long getNetBytesSent() {
return netBytesSent;
}
public long getEventTimeMillis() {
return eventTimeMillis;
}
@ -152,5 +127,21 @@ public class UsageNetworkVO {
public Long getNetworkId() {
return networkId;
}
}
public long getAggBytesReceived() {
return aggBytesReceived;
}
public void setAggBytesReceived(long aggBytesReceived) {
this.aggBytesReceived = aggBytesReceived;
}
public long getAggBytesSent() {
return aggBytesSent;
}
public void setAggBytesSent(long aggBytesSent) {
this.aggBytesSent = aggBytesSent;
}
}

View File

@ -47,10 +47,11 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage
private static final String DELETE_ALL = "DELETE FROM cloud_usage";
private static final String DELETE_ALL_BY_ACCOUNTID = "DELETE FROM cloud_usage WHERE account_id = ?";
private static final String INSERT_ACCOUNT = "INSERT INTO cloud_usage.account (id, account_name, type, domain_id, removed, cleanup_needed) VALUES (?,?,?,?,?,?)";
private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received, net_bytes_sent, current_bytes_received, current_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?)";
private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received," +
" net_bytes_sent, current_bytes_received, current_bytes_sent, agg_bytes_received, agg_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?, ?, ?)";
private static final String UPDATE_ACCOUNT = "UPDATE cloud_usage.account SET account_name=?, removed=? WHERE id=?";
private static final String UPDATE_USER_STATS = "UPDATE cloud_usage.user_statistics SET net_bytes_received=?, net_bytes_sent=?, current_bytes_received=?, current_bytes_sent=? WHERE id=?";
private static final String UPDATE_USER_STATS = "UPDATE cloud_usage.user_statistics SET net_bytes_received=?, net_bytes_sent=?, current_bytes_received=?, current_bytes_sent=?, agg_bytes_received=?, agg_bytes_sent=? WHERE id=?";
private static final String GET_LAST_ACCOUNT = "SELECT id FROM cloud_usage.account ORDER BY id DESC LIMIT 1";
private static final String GET_LAST_USER_STATS = "SELECT id FROM cloud_usage.user_statistics ORDER BY id DESC LIMIT 1";
@ -176,7 +177,9 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage
pstmt.setLong(8, userStat.getNetBytesReceived());
pstmt.setLong(9, userStat.getNetBytesSent());
pstmt.setLong(10, userStat.getCurrentBytesReceived());
pstmt.setLong(11, userStat.getCurrentBytesSent());
pstmt.setLong(11, userStat.getCurrentBytesSent());
pstmt.setLong(12, userStat.getAggBytesReceived());
pstmt.setLong(13, userStat.getAggBytesSent());
pstmt.addBatch();
}
pstmt.executeBatch();
@ -200,8 +203,10 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage
pstmt.setLong(1, userStat.getNetBytesReceived());
pstmt.setLong(2, userStat.getNetBytesSent());
pstmt.setLong(3, userStat.getCurrentBytesReceived());
pstmt.setLong(4, userStat.getCurrentBytesSent());
pstmt.setLong(5, userStat.getId());
pstmt.setLong(4, userStat.getCurrentBytesSent());
pstmt.setLong(5, userStat.getAggBytesReceived());
pstmt.setLong(6, userStat.getAggBytesSent());
pstmt.setLong(7, userStat.getId());
pstmt.addBatch();
}
pstmt.executeBatch();

View File

@ -35,8 +35,7 @@ import com.cloud.utils.db.Transaction;
@Local(value={UsageNetworkDao.class})
public class UsageNetworkDaoImpl extends GenericDaoBase<UsageNetworkVO, Long> implements UsageNetworkDao {
private static final Logger s_logger = Logger.getLogger(UsageVMInstanceDaoImpl.class.getName());
private static final String SELECT_LATEST_STATS = "SELECT u.account_id, u.zone_id, u.host_id, u.host_type, u.network_id, u.bytes_sent, u.bytes_received, u.net_bytes_received, u.net_bytes_sent, " +
"u.current_bytes_received, u.current_bytes_sent, u.event_time_millis " +
private static final String SELECT_LATEST_STATS = "SELECT u.account_id, u.zone_id, u.host_id, u.host_type, u.network_id, u.bytes_sent, u.bytes_received, u.agg_bytes_received, u.agg_bytes_sent, u.event_time_millis " +
"FROM cloud_usage.usage_network u INNER JOIN (SELECT netusage.account_id as acct_id, netusage.zone_id as z_id, max(netusage.event_time_millis) as max_date " +
"FROM cloud_usage.usage_network netusage " +
"GROUP BY netusage.account_id, netusage.zone_id " +
@ -63,17 +62,13 @@ public class UsageNetworkDaoImpl extends GenericDaoBase<UsageNetworkVO, Long> im
Long networkId = rs.getLong(5);
long bytesSent = rs.getLong(6);
long bytesReceived = rs.getLong(7);
long netBytesReceived = rs.getLong(8);
long netBytesSent = rs.getLong(9);
long currentBytesReceived = rs.getLong(10);
long currentBytesSent = rs.getLong(11);
long eventTimeMillis = rs.getLong(12);
long aggBytesReceived = rs.getLong(8);
long aggBytesSent = rs.getLong(9);
long eventTimeMillis = rs.getLong(10);
if(hostId != 0){
returnMap.put(zoneId + "-" + accountId+ "-Host-" + hostId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, netBytesReceived, netBytesSent,
currentBytesReceived, currentBytesSent, eventTimeMillis));
returnMap.put(zoneId + "-" + accountId+ "-Host-" + hostId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, aggBytesReceived, aggBytesSent, eventTimeMillis));
} else {
returnMap.put(zoneId + "-" + accountId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, netBytesReceived, netBytesSent, currentBytesReceived,
currentBytesSent, eventTimeMillis));
returnMap.put(zoneId + "-" + accountId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, aggBytesReceived, aggBytesSent, eventTimeMillis));
}
}
return returnMap;

View File

@ -31,5 +31,7 @@ public interface UserStatisticsDao extends GenericDao<UserStatisticsVO, Long> {
List<UserStatisticsVO> listBy(long accountId);
List<UserStatisticsVO> listActiveAndRecentlyDeleted(Date minRemovedDate, int startIndex, int limit);
List<UserStatisticsVO> listActiveAndRecentlyDeleted(Date minRemovedDate, int startIndex, int limit);
boolean updateAggStats();
}

View File

@ -39,10 +39,11 @@ import com.cloud.utils.db.Transaction;
@Local(value={UserStatisticsDao.class})
public class UserStatisticsDaoImpl extends GenericDaoBase<UserStatisticsVO, Long> implements UserStatisticsDao {
private static final Logger s_logger = Logger.getLogger(UserStatisticsDaoImpl.class);
private static final String ACTIVE_AND_RECENTLY_DELETED_SEARCH = "SELECT us.id, us.data_center_id, us.account_id, us.public_ip_address, us.device_id, us.device_type, us.network_id, us.net_bytes_received, us.net_bytes_sent, us.current_bytes_received, us.current_bytes_sent " +
private static final String ACTIVE_AND_RECENTLY_DELETED_SEARCH = "SELECT us.id, us.data_center_id, us.account_id, us.public_ip_address, us.device_id, us.device_type, us.network_id, us.agg_bytes_received, us.agg_bytes_sent " +
"FROM user_statistics us, account a " +
"WHERE us.account_id = a.id AND (a.removed IS NULL OR a.removed >= ?) " +
"ORDER BY us.id";
"ORDER BY us.id";
private static final String UPDATE_AGG_STATS = "UPDATE user_statistics set agg_bytes_received = net_bytes_received + current_bytes_received , agg_bytes_sent = net_bytes_sent + current_bytes_sent";
private final SearchBuilder<UserStatisticsVO> AllFieldsSearch;
private final SearchBuilder<UserStatisticsVO> AccountSearch;
@ -111,5 +112,19 @@ public class UserStatisticsDaoImpl extends GenericDaoBase<UserStatisticsVO, Long
s_logger.error("error saving user stats to cloud_usage db", ex);
}
return userStats;
}
@Override
public boolean updateAggStats(){
Transaction txn = Transaction.currentTxn();
try {
String sql = UPDATE_AGG_STATS;
PreparedStatement pstmt = null;
pstmt = txn.prepareAutoCloseStatement(sql);
return pstmt.executeUpdate() > 0;
} catch (Exception ex) {
s_logger.error("error updating agg user stats", ex);
}
return false;
}
}

View File

@ -66,10 +66,8 @@ CREATE TABLE `cloud_usage`.`usage_network` (
`network_id` bigint unsigned,
`bytes_sent` bigint unsigned NOT NULL default '0',
`bytes_received` bigint unsigned NOT NULL default '0',
`net_bytes_received` bigint unsigned NOT NULL default '0',
`net_bytes_sent` bigint unsigned NOT NULL default '0',
`current_bytes_received` bigint unsigned NOT NULL default '0',
`current_bytes_sent` bigint unsigned NOT NULL default '0',
`agg_bytes_received` bigint unsigned NOT NULL default '0',
`agg_bytes_sent` bigint unsigned NOT NULL default '0',
`event_time_millis` bigint unsigned NOT NULL default '0',
PRIMARY KEY (`account_id`, `zone_id`, `host_id`, `event_time_millis`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@ -134,6 +132,8 @@ CREATE TABLE `cloud_usage`.`user_statistics` (
`net_bytes_sent` bigint unsigned NOT NULL default '0',
`current_bytes_received` bigint unsigned NOT NULL default '0',
`current_bytes_sent` bigint unsigned NOT NULL default '0',
`agg_bytes_received` bigint unsigned NOT NULL default '0',
`agg_bytes_sent` bigint unsigned NOT NULL default '0',
PRIMARY KEY (`id`),
UNIQUE KEY (`account_id`, `data_center_id`, `public_ip_address`, `device_id`, `device_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

View File

@ -833,6 +833,8 @@ CREATE TABLE `cloud`.`user_statistics` (
`net_bytes_sent` bigint unsigned NOT NULL default '0',
`current_bytes_received` bigint unsigned NOT NULL default '0',
`current_bytes_sent` bigint unsigned NOT NULL default '0',
`agg_bytes_received` bigint unsigned NOT NULL default '0',
`agg_bytes_sent` bigint unsigned NOT NULL default '0',
PRIMARY KEY (`id`),
UNIQUE KEY (`account_id`, `data_center_id`, `public_ip_address`, `device_id`, `device_type`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

View File

@ -0,0 +1,17 @@
--;
-- Premium Schema upgrade from 2.2.13 to 3.0;
--;
ALTER TABLE `cloud_usage`.`user_statistics` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0';
ALTER TABLE `cloud_usage`.`user_statistics` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0';
ALTER TABLE `cloud_usage`.`usage_network` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0';
ALTER TABLE `cloud_usage`.`usage_network` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0';
update `cloud_usage`.`usage_network` set agg_bytes_received = net_bytes_received + current_bytes_received, agg_bytes_sent = net_bytes_sent + current_bytes_sent;
ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `net_bytes_received`;
ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `net_bytes_sent`;
ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `current_bytes_received`;
ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `current_bytes_sent`;

View File

@ -1,5 +1,5 @@
--;
-- Schema upgrade from 2.2.x to 3.0;
-- Schema upgrade from 2.2.13 to 3.0;
--;
ALTER TABLE `cloud`.`template_host_ref` DROP COLUMN `pool_id`;
@ -116,4 +116,8 @@ update configuration set name = 'pod.privateip.capacity.notificationthreshold' ,
ALTER TABLE `cloud`.`domain_router` ADD COLUMN `template_version` varchar(100) COMMENT 'template version' AFTER role;
ALTER TABLE `cloud`.`domain_router` ADD COLUMN `scripts_version` varchar(100) COMMENT 'scripts version' AFTER template_version;
ALTER TABLE `cloud`.`alert` ADD `cluster_id` bigint unsigned;
DELETE from `cloud`.`op_host_capacity` where capacity_type in (2,4,6);
ALTER TABLE `cloud`.`user_statistics` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0';
ALTER TABLE `cloud`.`user_statistics` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0';

View File

@ -266,7 +266,9 @@ public class UsageManagerImpl implements UsageManager, Runnable {
public boolean stop() {
m_heartbeat.cancel(true);
m_scheduledFuture.cancel(true);
m_sanity.cancel(true);
if(m_sanity != null){
m_sanity.cancel(true);
}
return true;
}
@ -539,10 +541,8 @@ public class UsageManagerImpl implements UsageManager, Runnable {
userStat.getDeviceId(), userStat.getDeviceType(), userStat.getNetworkId());
}
hostAggregatedStat.setNetBytesSent(hostAggregatedStat.getNetBytesSent() + userStat.getNetBytesSent());
hostAggregatedStat.setNetBytesReceived(hostAggregatedStat.getNetBytesReceived() + userStat.getNetBytesReceived());
hostAggregatedStat.setCurrentBytesSent(hostAggregatedStat.getCurrentBytesSent() + userStat.getCurrentBytesSent());
hostAggregatedStat.setCurrentBytesReceived(hostAggregatedStat.getCurrentBytesReceived() + userStat.getCurrentBytesReceived());
hostAggregatedStat.setAggBytesSent(hostAggregatedStat.getAggBytesSent() + userStat.getAggBytesSent());
hostAggregatedStat.setAggBytesReceived(hostAggregatedStat.getAggBytesReceived() + userStat.getAggBytesReceived());
aggregatedStats.put(hostKey, hostAggregatedStat);
}
}
@ -566,11 +566,6 @@ public class UsageManagerImpl implements UsageManager, Runnable {
s_logger.debug("created network stats helper entries for " + numAcctsProcessed + " accts");
}
if (s_logger.isDebugEnabled()) {
s_logger.debug("deleting old network stats helper entries older than " + deleteOldStatsTimeMillis);
}
m_usageNetworkDao.deleteOldStats(deleteOldStatsTimeMillis);
// commit the helper records, then start a new transaction
usageTxn.commit();
usageTxn.start();
@ -952,21 +947,21 @@ public class UsageManagerImpl implements UsageManager, Runnable {
long currentAccountedBytesReceived = 0L;
if (usageNetworkStats != null) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("getting current accounted bytes for... accountId: " + usageNetworkStats.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; cbr: " + usageNetworkStats.getCurrentBytesReceived() +
"; cbs: " + usageNetworkStats.getCurrentBytesSent() + "; nbr: " + usageNetworkStats.getNetBytesReceived() + "; nbs: " + usageNetworkStats.getNetBytesSent());
s_logger.debug("getting current accounted bytes for... accountId: " + usageNetworkStats.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; abr: " + usageNetworkStats.getAggBytesReceived() +
"; abs: " + usageNetworkStats.getAggBytesSent());
}
currentAccountedBytesSent = (usageNetworkStats.getCurrentBytesSent() + usageNetworkStats.getNetBytesSent());
currentAccountedBytesReceived = (usageNetworkStats.getCurrentBytesReceived() + usageNetworkStats.getNetBytesReceived());
currentAccountedBytesSent = usageNetworkStats.getAggBytesSent();
currentAccountedBytesReceived = usageNetworkStats.getAggBytesReceived();
}
long bytesSent = (userStat.getCurrentBytesSent() + userStat.getNetBytesSent()) - currentAccountedBytesSent;
long bytesReceived = (userStat.getCurrentBytesReceived() + userStat.getNetBytesReceived()) - currentAccountedBytesReceived;
long bytesSent = userStat.getAggBytesSent() - currentAccountedBytesSent;
long bytesReceived = userStat.getAggBytesReceived() - currentAccountedBytesReceived;
if (bytesSent < 0) {
s_logger.warn("Calculated negative value for bytes sent: " + bytesSent + ", user stats say: " + (userStat.getCurrentBytesSent() + userStat.getNetBytesSent()) + ", previous network usage was: " + currentAccountedBytesSent);
s_logger.warn("Calculated negative value for bytes sent: " + bytesSent + ", user stats say: " + userStat.getAggBytesSent() + ", previous network usage was: " + currentAccountedBytesSent);
bytesSent = 0;
}
if (bytesReceived < 0) {
s_logger.warn("Calculated negative value for bytes received: " + bytesReceived + ", user stats say: " + (userStat.getCurrentBytesReceived() + userStat.getNetBytesReceived()) + ", previous network usage was: " + currentAccountedBytesReceived);
s_logger.warn("Calculated negative value for bytes received: " + bytesReceived + ", user stats say: " + userStat.getAggBytesReceived() + ", previous network usage was: " + currentAccountedBytesReceived);
bytesReceived = 0;
}
@ -977,11 +972,10 @@ public class UsageManagerImpl implements UsageManager, Runnable {
}
UsageNetworkVO usageNetworkVO = new UsageNetworkVO(userStat.getAccountId(), userStat.getDataCenterId(), hostId, userStat.getDeviceType(), userStat.getNetworkId(), bytesSent, bytesReceived,
userStat.getNetBytesReceived(), userStat.getNetBytesSent(),
userStat.getCurrentBytesReceived(), userStat.getCurrentBytesSent(), timestamp);
userStat.getAggBytesReceived(), userStat.getAggBytesSent(), timestamp);
if (s_logger.isDebugEnabled()) {
s_logger.debug("creating networkHelperEntry... accountId: " + userStat.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; cbr: " + userStat.getCurrentBytesReceived() + "; cbs: " + userStat.getCurrentBytesSent() +
"; nbr: " + userStat.getNetBytesReceived() + "; nbs: " + userStat.getNetBytesSent() + "; curABS: " + currentAccountedBytesSent + "; curABR: " + currentAccountedBytesReceived + "; ubs: " + bytesSent + "; ubr: " + bytesReceived);
s_logger.debug("creating networkHelperEntry... accountId: " + userStat.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; abr: " + userStat.getAggBytesReceived() + "; abs: " + userStat.getAggBytesSent() +
"; curABS: " + currentAccountedBytesSent + "; curABR: " + currentAccountedBytesReceived + "; ubs: " + bytesSent + "; ubr: " + bytesReceived);
}
m_usageNetworkDao.persist(usageNetworkVO);
}