From 284ee3909ed1e6769cccf3577d5c46ed13651436 Mon Sep 17 00:00:00 2001 From: kishan Date: Mon, 31 Oct 2011 17:12:12 +0530 Subject: [PATCH] bug 9842: Added task to update user stats agg bytes in sync with aggregation range status 9842: resolved fixed --- core/src/com/cloud/user/UserStatisticsVO.java | 26 ++++++- .../VirtualNetworkApplianceManagerImpl.java | 68 ++++++++++++++++++- .../PremiumDatabaseUpgradeChecker.java | 34 +++++----- .../upgrade/dao/Upgrade2213to30Premium.java | 65 ++++++++++++++++++ .../src/com/cloud/usage/UsageNetworkVO.java | 57 +++++++--------- .../src/com/cloud/usage/dao/UsageDaoImpl.java | 15 ++-- .../cloud/usage/dao/UsageNetworkDaoImpl.java | 17 ++--- .../com/cloud/user/dao/UserStatisticsDao.java | 4 +- .../cloud/user/dao/UserStatisticsDaoImpl.java | 19 +++++- setup/db/create-schema-premium.sql | 8 +-- setup/db/create-schema.sql | 2 + setup/db/db/schema-2213to30-premium.sql | 17 +++++ setup/db/db/schema-2213to30.sql | 6 +- .../src/com/cloud/usage/UsageManagerImpl.java | 38 +++++------ 14 files changed, 277 insertions(+), 99 deletions(-) create mode 100644 server/src/com/cloud/upgrade/dao/Upgrade2213to30Premium.java create mode 100755 setup/db/db/schema-2213to30-premium.sql diff --git a/core/src/com/cloud/user/UserStatisticsVO.java b/core/src/com/cloud/user/UserStatisticsVO.java index 1636c3ef2d8..d35552e3351 100644 --- a/core/src/com/cloud/user/UserStatisticsVO.java +++ b/core/src/com/cloud/user/UserStatisticsVO.java @@ -62,7 +62,13 @@ public class UserStatisticsVO { private long currentBytesReceived; @Column(name="current_bytes_sent") - private long currentBytesSent; + private long currentBytesSent; + + @Column(name="agg_bytes_received") + private long aggBytesReceived; + + @Column(name="agg_bytes_sent") + private long aggBytesSent; protected UserStatisticsVO() { } @@ -77,7 +83,7 @@ public class UserStatisticsVO { this.netBytesReceived = 0; this.netBytesSent = 0; this.currentBytesReceived = 0; - this.currentBytesSent = 0; + this.currentBytesSent = 0; } public long getAccountId() { @@ -139,5 +145,21 @@ public class UserStatisticsVO { public void setNetBytesSent(long netBytesSent) { this.netBytesSent = netBytesSent; } + + public long getAggBytesReceived() { + return aggBytesReceived; + } + + public void setAggBytesReceived(long aggBytesReceived) { + this.aggBytesReceived = aggBytesReceived; + } + + public long getAggBytesSent() { + return aggBytesSent; + } + + public void setAggBytesSent(long aggBytesSent) { + this.aggBytesSent = aggBytesSent; + } } diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index eb90618a0ab..cc4720c153c 100755 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -18,13 +18,16 @@ package com.cloud.network.router; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collections; import java.util.Comparator; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -337,10 +340,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian private boolean _disable_rp_filter = false; int _routerExtraPublicNics = 2; + private int _usageAggregationRange = 1440; + private String _usageTimeZone = "GMT"; private long mgmtSrvrId = MacAddress.getMacAddress().toLong(); ScheduledExecutorService _executor; ScheduledExecutorService _checkExecutor; + ScheduledExecutorService _networkStatsUpdateExecutor; Account _systemAcct; @@ -592,6 +598,7 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterMonitor")); _checkExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("RouterStatusMonitor")); + _networkStatsUpdateExecutor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("NetworkStatsUpdater")); final ComponentLocator locator = ComponentLocator.getCurrentLocator(); @@ -645,6 +652,13 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian _systemAcct = _accountService.getSystemAccount(); + String aggregationRange = configs.get("usage.stats.job.aggregation.range"); + _usageAggregationRange = NumbersUtil.parseInt(aggregationRange, 1440); + _usageTimeZone = configs.get("usage.aggregation.timezone"); + if(_usageTimeZone == null){ + _usageTimeZone = "GMT"; + } + _agentMgr.registerForHostEvents(this, true, false, false); s_logger.info("DomainRouterManager is configured."); @@ -661,10 +675,41 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian public boolean start() { if (_routerStatsInterval > 0){ _executor.scheduleAtFixedRate(new NetworkUsageTask(), _routerStatsInterval, _routerStatsInterval, TimeUnit.SECONDS); + + //Schedule Network stats update task + TimeZone usageTimezone = TimeZone.getTimeZone(_usageTimeZone); + Calendar cal = Calendar.getInstance(usageTimezone); + cal.setTime(new Date()); + long endDate = 0; + int HOURLY_TIME = 60; + final int DAILY_TIME = 60 * 24; + if (_usageAggregationRange == DAILY_TIME) { + cal.roll(Calendar.DAY_OF_YEAR, false); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.roll(Calendar.DAY_OF_YEAR, true); + cal.add(Calendar.MILLISECOND, -1); + endDate = cal.getTime().getTime(); + } else if (_usageAggregationRange == HOURLY_TIME) { + cal.roll(Calendar.HOUR_OF_DAY, false); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + cal.roll(Calendar.HOUR_OF_DAY, true); + cal.add(Calendar.MILLISECOND, -1); + endDate = cal.getTime().getTime(); + } else { + endDate = cal.getTime().getTime(); + } + + _networkStatsUpdateExecutor.scheduleAtFixedRate(new NetworkStatsUpdateTask(), (endDate - System.currentTimeMillis()), (_usageAggregationRange * 60 * 1000), TimeUnit.MILLISECONDS); }else{ s_logger.debug("router.stats.interval - " + _routerStatsInterval+ " so not scheduling the router stats thread"); } - _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _checkRouterInterval, _checkRouterInterval, TimeUnit.SECONDS); + _checkExecutor.scheduleAtFixedRate(new CheckRouterTask(), _checkRouterInterval, _checkRouterInterval, TimeUnit.SECONDS); + return true; } @@ -791,6 +836,27 @@ public class VirtualNetworkApplianceManagerImpl implements VirtualNetworkApplian } } + protected class NetworkStatsUpdateTask implements Runnable { + + public NetworkStatsUpdateTask() { + } + + @Override + public void run() { + try{ + if(_statsDao.updateAggStats()){ + s_logger.debug("Succussfully updated aggregate network stats"); + } else { + s_logger.debug("Failed to update aggregate network stats"); + } + } catch (Exception e){ + s_logger.debug("Failed to update aggregate network stats", e); + } + } + + } + + protected void updateRoutersRedundantState(List routers) { boolean updated = false; for (DomainRouterVO router : routers) { diff --git a/server/src/com/cloud/upgrade/PremiumDatabaseUpgradeChecker.java b/server/src/com/cloud/upgrade/PremiumDatabaseUpgradeChecker.java index fbf96770890..7e74e4dbee2 100755 --- a/server/src/com/cloud/upgrade/PremiumDatabaseUpgradeChecker.java +++ b/server/src/com/cloud/upgrade/PremiumDatabaseUpgradeChecker.java @@ -27,6 +27,7 @@ import com.cloud.upgrade.dao.Upgrade218to22Premium; import com.cloud.upgrade.dao.Upgrade2210to2211; import com.cloud.upgrade.dao.Upgrade2211to2212Premium; import com.cloud.upgrade.dao.Upgrade2212to2213; +import com.cloud.upgrade.dao.Upgrade2213to30Premium; import com.cloud.upgrade.dao.Upgrade221to222Premium; import com.cloud.upgrade.dao.Upgrade222to224Premium; import com.cloud.upgrade.dao.Upgrade224to225; @@ -34,7 +35,6 @@ import com.cloud.upgrade.dao.Upgrade225to226; import com.cloud.upgrade.dao.Upgrade227to228Premium; import com.cloud.upgrade.dao.Upgrade228to229; import com.cloud.upgrade.dao.Upgrade229to2210; -import com.cloud.upgrade.dao.Upgrade2213to30; import com.cloud.upgrade.dao.UpgradeSnapshot217to224; import com.cloud.upgrade.dao.UpgradeSnapshot223to224; import com.cloud.upgrade.dao.VersionDaoImpl; @@ -46,23 +46,23 @@ public class PremiumDatabaseUpgradeChecker extends DatabaseUpgradeChecker { public PremiumDatabaseUpgradeChecker() { _dao = ComponentLocator.inject(VersionDaoImpl.class); _upgradeMap.put("2.1.7", new DbUpgrade[] { new Upgrade217to218(), new Upgrade218to22Premium(), new Upgrade221to222Premium(), new UpgradeSnapshot217to224(), new Upgrade222to224Premium(), - new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); + new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); _upgradeMap.put("2.1.8", new DbUpgrade[] { new Upgrade218to22Premium(), new Upgrade221to222Premium(), new UpgradeSnapshot217to224(), new Upgrade222to224Premium(), - new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213() , new Upgrade2213to30()}); + new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213() , new Upgrade2213to30Premium()}); _upgradeMap.put("2.1.9", new DbUpgrade[] { new Upgrade218to22Premium(), new Upgrade221to222Premium(), new UpgradeSnapshot217to224(), new Upgrade222to224Premium(), - new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.1", new DbUpgrade[] { new Upgrade221to222Premium(), new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.2", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.3", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.4", new DbUpgrade[] { new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.5", new DbUpgrade[] { new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.6", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.7", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.8", new DbUpgrade[] { new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.9", new DbUpgrade[] { new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.10", new DbUpgrade[] { new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.11", new DbUpgrade[] { new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.12", new DbUpgrade[] { new Upgrade2212to2213(), new Upgrade2213to30()}); - _upgradeMap.put("2.2.13", new DbUpgrade[] { new Upgrade2213to30()}); + new Upgrade218to224DomainVlans(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.1", new DbUpgrade[] { new Upgrade221to222Premium(), new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.2", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.3", new DbUpgrade[] { new Upgrade222to224Premium(), new UpgradeSnapshot223to224(), new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.4", new DbUpgrade[] { new Upgrade224to225(), new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.5", new DbUpgrade[] { new Upgrade225to226(), new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.6", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.7", new DbUpgrade[] { new Upgrade227to228Premium(), new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.8", new DbUpgrade[] { new Upgrade228to229(), new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.9", new DbUpgrade[] { new Upgrade229to2210(), new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.10", new DbUpgrade[] { new Upgrade2210to2211(), new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.11", new DbUpgrade[] { new Upgrade2211to2212Premium(), new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.12", new DbUpgrade[] { new Upgrade2212to2213(), new Upgrade2213to30Premium()}); + _upgradeMap.put("2.2.13", new DbUpgrade[] { new Upgrade2213to30Premium()}); } } diff --git a/server/src/com/cloud/upgrade/dao/Upgrade2213to30Premium.java b/server/src/com/cloud/upgrade/dao/Upgrade2213to30Premium.java new file mode 100644 index 00000000000..c90a778eb49 --- /dev/null +++ b/server/src/com/cloud/upgrade/dao/Upgrade2213to30Premium.java @@ -0,0 +1,65 @@ +/** + * Copyright (C) 2011 Citrix Systems, Inc. All rights reserved + * + * This software is licensed under the GNU General Public License v3 or later. + * + * It is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or any later version. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ +package com.cloud.upgrade.dao; + +import java.io.File; +import java.sql.Connection; + +import org.apache.log4j.Logger; + +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; + +public class Upgrade2213to30Premium implements DbUpgrade { + final static Logger s_logger = Logger.getLogger(Upgrade2213to30Premium.class); + + @Override + public String[] getUpgradableVersionRange() { + return new String[] { "2.2.13", "3.0.0"}; + } + + @Override + public String getUpgradedVersion() { + return "3.0.0"; + } + + @Override + public boolean supportsRollingUpgrade() { + return true; + } + + @Override + public File[] getPrepareScripts() { + String script = Script.findScript("", "db/db/schema-2213to30-premium.sql"); + if (script == null) { + throw new CloudRuntimeException("Unable to find db/schema-2213to30-premium.sql"); + } + + return new File[] { new File(script) }; + } + + @Override + public void performDataMigration(Connection conn) { + } + + @Override + public File[] getCleanupScripts() { + return null; + } + +} diff --git a/server/src/com/cloud/usage/UsageNetworkVO.java b/server/src/com/cloud/usage/UsageNetworkVO.java index 91b5546c38c..d592be5c6ad 100644 --- a/server/src/com/cloud/usage/UsageNetworkVO.java +++ b/server/src/com/cloud/usage/UsageNetworkVO.java @@ -50,26 +50,19 @@ public class UsageNetworkVO { @Column(name="bytes_received") private long bytesReceived; - @Column(name="net_bytes_received") - private long netBytesReceived; + @Column(name="agg_bytes_received") + private long aggBytesReceived; - @Column(name="net_bytes_sent") - private long netBytesSent; + @Column(name="agg_bytes_sent") + private long aggBytesSent; - @Column(name="current_bytes_received") - private long currentBytesReceived; - - @Column(name="current_bytes_sent") - private long currentBytesSent; - @Column(name="event_time_millis") private long eventTimeMillis = 0; protected UsageNetworkVO() { } - public UsageNetworkVO(Long accountId, long zoneId, long hostId, String hostType, Long networkId, long bytesSent, long bytesReceived, long netBytesReceived, long netBytesSent, long currentBytesReceived, - long currentBytesSent, long eventTimeMillis) { + public UsageNetworkVO(Long accountId, long zoneId, long hostId, String hostType, Long networkId, long bytesSent, long bytesReceived, long aggBytesReceived, long aggBytesSent, long eventTimeMillis) { this.accountId = accountId; this.zoneId = zoneId; this.hostId = hostId; @@ -77,10 +70,8 @@ public class UsageNetworkVO { this.networkId = networkId; this.bytesSent = bytesSent; this.bytesReceived = bytesReceived; - this.netBytesReceived = netBytesReceived; - this.netBytesSent = netBytesSent; - this.currentBytesReceived = currentBytesReceived; - this.currentBytesSent = currentBytesSent; + this.aggBytesReceived = aggBytesReceived; + this.aggBytesSent = aggBytesSent; this.eventTimeMillis = eventTimeMillis; } @@ -115,22 +106,6 @@ public class UsageNetworkVO { this.bytesReceived = bytesReceived; } - public long getCurrentBytesReceived() { - return currentBytesReceived; - } - - public long getCurrentBytesSent() { - return currentBytesSent; - } - - public long getNetBytesReceived() { - return netBytesReceived; - } - - public long getNetBytesSent() { - return netBytesSent; - } - public long getEventTimeMillis() { return eventTimeMillis; } @@ -152,5 +127,21 @@ public class UsageNetworkVO { public Long getNetworkId() { return networkId; - } + } + + public long getAggBytesReceived() { + return aggBytesReceived; + } + + public void setAggBytesReceived(long aggBytesReceived) { + this.aggBytesReceived = aggBytesReceived; + } + + public long getAggBytesSent() { + return aggBytesSent; + } + + public void setAggBytesSent(long aggBytesSent) { + this.aggBytesSent = aggBytesSent; + } } diff --git a/server/src/com/cloud/usage/dao/UsageDaoImpl.java b/server/src/com/cloud/usage/dao/UsageDaoImpl.java index c93ba600d83..43d76389783 100644 --- a/server/src/com/cloud/usage/dao/UsageDaoImpl.java +++ b/server/src/com/cloud/usage/dao/UsageDaoImpl.java @@ -47,10 +47,11 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage private static final String DELETE_ALL = "DELETE FROM cloud_usage"; private static final String DELETE_ALL_BY_ACCOUNTID = "DELETE FROM cloud_usage WHERE account_id = ?"; private static final String INSERT_ACCOUNT = "INSERT INTO cloud_usage.account (id, account_name, type, domain_id, removed, cleanup_needed) VALUES (?,?,?,?,?,?)"; - private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received, net_bytes_sent, current_bytes_received, current_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?)"; + private static final String INSERT_USER_STATS = "INSERT INTO cloud_usage.user_statistics (id, data_center_id, account_id, public_ip_address, device_id, device_type, network_id, net_bytes_received," + + " net_bytes_sent, current_bytes_received, current_bytes_sent, agg_bytes_received, agg_bytes_sent) VALUES (?,?,?,?,?,?,?,?,?,?, ?, ?, ?)"; private static final String UPDATE_ACCOUNT = "UPDATE cloud_usage.account SET account_name=?, removed=? WHERE id=?"; - private static final String UPDATE_USER_STATS = "UPDATE cloud_usage.user_statistics SET net_bytes_received=?, net_bytes_sent=?, current_bytes_received=?, current_bytes_sent=? WHERE id=?"; + private static final String UPDATE_USER_STATS = "UPDATE cloud_usage.user_statistics SET net_bytes_received=?, net_bytes_sent=?, current_bytes_received=?, current_bytes_sent=?, agg_bytes_received=?, agg_bytes_sent=? WHERE id=?"; private static final String GET_LAST_ACCOUNT = "SELECT id FROM cloud_usage.account ORDER BY id DESC LIMIT 1"; private static final String GET_LAST_USER_STATS = "SELECT id FROM cloud_usage.user_statistics ORDER BY id DESC LIMIT 1"; @@ -176,7 +177,9 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage pstmt.setLong(8, userStat.getNetBytesReceived()); pstmt.setLong(9, userStat.getNetBytesSent()); pstmt.setLong(10, userStat.getCurrentBytesReceived()); - pstmt.setLong(11, userStat.getCurrentBytesSent()); + pstmt.setLong(11, userStat.getCurrentBytesSent()); + pstmt.setLong(12, userStat.getAggBytesReceived()); + pstmt.setLong(13, userStat.getAggBytesSent()); pstmt.addBatch(); } pstmt.executeBatch(); @@ -200,8 +203,10 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage pstmt.setLong(1, userStat.getNetBytesReceived()); pstmt.setLong(2, userStat.getNetBytesSent()); pstmt.setLong(3, userStat.getCurrentBytesReceived()); - pstmt.setLong(4, userStat.getCurrentBytesSent()); - pstmt.setLong(5, userStat.getId()); + pstmt.setLong(4, userStat.getCurrentBytesSent()); + pstmt.setLong(5, userStat.getAggBytesReceived()); + pstmt.setLong(6, userStat.getAggBytesSent()); + pstmt.setLong(7, userStat.getId()); pstmt.addBatch(); } pstmt.executeBatch(); diff --git a/server/src/com/cloud/usage/dao/UsageNetworkDaoImpl.java b/server/src/com/cloud/usage/dao/UsageNetworkDaoImpl.java index 7e17cb87200..6991af75458 100644 --- a/server/src/com/cloud/usage/dao/UsageNetworkDaoImpl.java +++ b/server/src/com/cloud/usage/dao/UsageNetworkDaoImpl.java @@ -35,8 +35,7 @@ import com.cloud.utils.db.Transaction; @Local(value={UsageNetworkDao.class}) public class UsageNetworkDaoImpl extends GenericDaoBase implements UsageNetworkDao { private static final Logger s_logger = Logger.getLogger(UsageVMInstanceDaoImpl.class.getName()); - private static final String SELECT_LATEST_STATS = "SELECT u.account_id, u.zone_id, u.host_id, u.host_type, u.network_id, u.bytes_sent, u.bytes_received, u.net_bytes_received, u.net_bytes_sent, " + - "u.current_bytes_received, u.current_bytes_sent, u.event_time_millis " + + private static final String SELECT_LATEST_STATS = "SELECT u.account_id, u.zone_id, u.host_id, u.host_type, u.network_id, u.bytes_sent, u.bytes_received, u.agg_bytes_received, u.agg_bytes_sent, u.event_time_millis " + "FROM cloud_usage.usage_network u INNER JOIN (SELECT netusage.account_id as acct_id, netusage.zone_id as z_id, max(netusage.event_time_millis) as max_date " + "FROM cloud_usage.usage_network netusage " + "GROUP BY netusage.account_id, netusage.zone_id " + @@ -63,17 +62,13 @@ public class UsageNetworkDaoImpl extends GenericDaoBase im Long networkId = rs.getLong(5); long bytesSent = rs.getLong(6); long bytesReceived = rs.getLong(7); - long netBytesReceived = rs.getLong(8); - long netBytesSent = rs.getLong(9); - long currentBytesReceived = rs.getLong(10); - long currentBytesSent = rs.getLong(11); - long eventTimeMillis = rs.getLong(12); + long aggBytesReceived = rs.getLong(8); + long aggBytesSent = rs.getLong(9); + long eventTimeMillis = rs.getLong(10); if(hostId != 0){ - returnMap.put(zoneId + "-" + accountId+ "-Host-" + hostId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, netBytesReceived, netBytesSent, - currentBytesReceived, currentBytesSent, eventTimeMillis)); + returnMap.put(zoneId + "-" + accountId+ "-Host-" + hostId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, aggBytesReceived, aggBytesSent, eventTimeMillis)); } else { - returnMap.put(zoneId + "-" + accountId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, netBytesReceived, netBytesSent, currentBytesReceived, - currentBytesSent, eventTimeMillis)); + returnMap.put(zoneId + "-" + accountId, new UsageNetworkVO(accountId, zoneId, hostId, hostType, networkId, bytesSent, bytesReceived, aggBytesReceived, aggBytesSent, eventTimeMillis)); } } return returnMap; diff --git a/server/src/com/cloud/user/dao/UserStatisticsDao.java b/server/src/com/cloud/user/dao/UserStatisticsDao.java index 5566b2dff49..a400f736fd3 100644 --- a/server/src/com/cloud/user/dao/UserStatisticsDao.java +++ b/server/src/com/cloud/user/dao/UserStatisticsDao.java @@ -31,5 +31,7 @@ public interface UserStatisticsDao extends GenericDao { List listBy(long accountId); - List listActiveAndRecentlyDeleted(Date minRemovedDate, int startIndex, int limit); + List listActiveAndRecentlyDeleted(Date minRemovedDate, int startIndex, int limit); + + boolean updateAggStats(); } diff --git a/server/src/com/cloud/user/dao/UserStatisticsDaoImpl.java b/server/src/com/cloud/user/dao/UserStatisticsDaoImpl.java index 7df6c9d57b8..298be81468e 100644 --- a/server/src/com/cloud/user/dao/UserStatisticsDaoImpl.java +++ b/server/src/com/cloud/user/dao/UserStatisticsDaoImpl.java @@ -39,10 +39,11 @@ import com.cloud.utils.db.Transaction; @Local(value={UserStatisticsDao.class}) public class UserStatisticsDaoImpl extends GenericDaoBase implements UserStatisticsDao { private static final Logger s_logger = Logger.getLogger(UserStatisticsDaoImpl.class); - private static final String ACTIVE_AND_RECENTLY_DELETED_SEARCH = "SELECT us.id, us.data_center_id, us.account_id, us.public_ip_address, us.device_id, us.device_type, us.network_id, us.net_bytes_received, us.net_bytes_sent, us.current_bytes_received, us.current_bytes_sent " + + private static final String ACTIVE_AND_RECENTLY_DELETED_SEARCH = "SELECT us.id, us.data_center_id, us.account_id, us.public_ip_address, us.device_id, us.device_type, us.network_id, us.agg_bytes_received, us.agg_bytes_sent " + "FROM user_statistics us, account a " + "WHERE us.account_id = a.id AND (a.removed IS NULL OR a.removed >= ?) " + - "ORDER BY us.id"; + "ORDER BY us.id"; + private static final String UPDATE_AGG_STATS = "UPDATE user_statistics set agg_bytes_received = net_bytes_received + current_bytes_received , agg_bytes_sent = net_bytes_sent + current_bytes_sent"; private final SearchBuilder AllFieldsSearch; private final SearchBuilder AccountSearch; @@ -111,5 +112,19 @@ public class UserStatisticsDaoImpl extends GenericDaoBase 0; + } catch (Exception ex) { + s_logger.error("error updating agg user stats", ex); + } + return false; } } diff --git a/setup/db/create-schema-premium.sql b/setup/db/create-schema-premium.sql index 2e58719304c..3fc33a2d442 100644 --- a/setup/db/create-schema-premium.sql +++ b/setup/db/create-schema-premium.sql @@ -66,10 +66,8 @@ CREATE TABLE `cloud_usage`.`usage_network` ( `network_id` bigint unsigned, `bytes_sent` bigint unsigned NOT NULL default '0', `bytes_received` bigint unsigned NOT NULL default '0', - `net_bytes_received` bigint unsigned NOT NULL default '0', - `net_bytes_sent` bigint unsigned NOT NULL default '0', - `current_bytes_received` bigint unsigned NOT NULL default '0', - `current_bytes_sent` bigint unsigned NOT NULL default '0', + `agg_bytes_received` bigint unsigned NOT NULL default '0', + `agg_bytes_sent` bigint unsigned NOT NULL default '0', `event_time_millis` bigint unsigned NOT NULL default '0', PRIMARY KEY (`account_id`, `zone_id`, `host_id`, `event_time_millis`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -134,6 +132,8 @@ CREATE TABLE `cloud_usage`.`user_statistics` ( `net_bytes_sent` bigint unsigned NOT NULL default '0', `current_bytes_received` bigint unsigned NOT NULL default '0', `current_bytes_sent` bigint unsigned NOT NULL default '0', + `agg_bytes_received` bigint unsigned NOT NULL default '0', + `agg_bytes_sent` bigint unsigned NOT NULL default '0', PRIMARY KEY (`id`), UNIQUE KEY (`account_id`, `data_center_id`, `public_ip_address`, `device_id`, `device_type`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 5e92323c90e..d8355d2a53f 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -833,6 +833,8 @@ CREATE TABLE `cloud`.`user_statistics` ( `net_bytes_sent` bigint unsigned NOT NULL default '0', `current_bytes_received` bigint unsigned NOT NULL default '0', `current_bytes_sent` bigint unsigned NOT NULL default '0', + `agg_bytes_received` bigint unsigned NOT NULL default '0', + `agg_bytes_sent` bigint unsigned NOT NULL default '0', PRIMARY KEY (`id`), UNIQUE KEY (`account_id`, `data_center_id`, `public_ip_address`, `device_id`, `device_type`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/setup/db/db/schema-2213to30-premium.sql b/setup/db/db/schema-2213to30-premium.sql new file mode 100755 index 00000000000..752814a4eb9 --- /dev/null +++ b/setup/db/db/schema-2213to30-premium.sql @@ -0,0 +1,17 @@ +--; +-- Premium Schema upgrade from 2.2.13 to 3.0; +--; + +ALTER TABLE `cloud_usage`.`user_statistics` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0'; +ALTER TABLE `cloud_usage`.`user_statistics` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0'; + +ALTER TABLE `cloud_usage`.`usage_network` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0'; +ALTER TABLE `cloud_usage`.`usage_network` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0'; + +update `cloud_usage`.`usage_network` set agg_bytes_received = net_bytes_received + current_bytes_received, agg_bytes_sent = net_bytes_sent + current_bytes_sent; + +ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `net_bytes_received`; +ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `net_bytes_sent`; +ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `current_bytes_received`; +ALTER TABLE `cloud_usage`.`usage_network` DROP COLUMN `current_bytes_sent`; + diff --git a/setup/db/db/schema-2213to30.sql b/setup/db/db/schema-2213to30.sql index bbbde540f5c..3eee6e2d16e 100755 --- a/setup/db/db/schema-2213to30.sql +++ b/setup/db/db/schema-2213to30.sql @@ -1,5 +1,5 @@ --; --- Schema upgrade from 2.2.x to 3.0; +-- Schema upgrade from 2.2.13 to 3.0; --; ALTER TABLE `cloud`.`template_host_ref` DROP COLUMN `pool_id`; @@ -116,4 +116,8 @@ update configuration set name = 'pod.privateip.capacity.notificationthreshold' , ALTER TABLE `cloud`.`domain_router` ADD COLUMN `template_version` varchar(100) COMMENT 'template version' AFTER role; ALTER TABLE `cloud`.`domain_router` ADD COLUMN `scripts_version` varchar(100) COMMENT 'scripts version' AFTER template_version; ALTER TABLE `cloud`.`alert` ADD `cluster_id` bigint unsigned; + DELETE from `cloud`.`op_host_capacity` where capacity_type in (2,4,6); + +ALTER TABLE `cloud`.`user_statistics` ADD COLUMN `agg_bytes_received` bigint unsigned NOT NULL default '0'; +ALTER TABLE `cloud`.`user_statistics` ADD COLUMN `agg_bytes_sent` bigint unsigned NOT NULL default '0'; diff --git a/usage/src/com/cloud/usage/UsageManagerImpl.java b/usage/src/com/cloud/usage/UsageManagerImpl.java index 684cbd07730..df77ac0f24b 100644 --- a/usage/src/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/com/cloud/usage/UsageManagerImpl.java @@ -266,7 +266,9 @@ public class UsageManagerImpl implements UsageManager, Runnable { public boolean stop() { m_heartbeat.cancel(true); m_scheduledFuture.cancel(true); - m_sanity.cancel(true); + if(m_sanity != null){ + m_sanity.cancel(true); + } return true; } @@ -539,10 +541,8 @@ public class UsageManagerImpl implements UsageManager, Runnable { userStat.getDeviceId(), userStat.getDeviceType(), userStat.getNetworkId()); } - hostAggregatedStat.setNetBytesSent(hostAggregatedStat.getNetBytesSent() + userStat.getNetBytesSent()); - hostAggregatedStat.setNetBytesReceived(hostAggregatedStat.getNetBytesReceived() + userStat.getNetBytesReceived()); - hostAggregatedStat.setCurrentBytesSent(hostAggregatedStat.getCurrentBytesSent() + userStat.getCurrentBytesSent()); - hostAggregatedStat.setCurrentBytesReceived(hostAggregatedStat.getCurrentBytesReceived() + userStat.getCurrentBytesReceived()); + hostAggregatedStat.setAggBytesSent(hostAggregatedStat.getAggBytesSent() + userStat.getAggBytesSent()); + hostAggregatedStat.setAggBytesReceived(hostAggregatedStat.getAggBytesReceived() + userStat.getAggBytesReceived()); aggregatedStats.put(hostKey, hostAggregatedStat); } } @@ -566,11 +566,6 @@ public class UsageManagerImpl implements UsageManager, Runnable { s_logger.debug("created network stats helper entries for " + numAcctsProcessed + " accts"); } - if (s_logger.isDebugEnabled()) { - s_logger.debug("deleting old network stats helper entries older than " + deleteOldStatsTimeMillis); - } - m_usageNetworkDao.deleteOldStats(deleteOldStatsTimeMillis); - // commit the helper records, then start a new transaction usageTxn.commit(); usageTxn.start(); @@ -952,21 +947,21 @@ public class UsageManagerImpl implements UsageManager, Runnable { long currentAccountedBytesReceived = 0L; if (usageNetworkStats != null) { if (s_logger.isDebugEnabled()) { - s_logger.debug("getting current accounted bytes for... accountId: " + usageNetworkStats.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; cbr: " + usageNetworkStats.getCurrentBytesReceived() + - "; cbs: " + usageNetworkStats.getCurrentBytesSent() + "; nbr: " + usageNetworkStats.getNetBytesReceived() + "; nbs: " + usageNetworkStats.getNetBytesSent()); + s_logger.debug("getting current accounted bytes for... accountId: " + usageNetworkStats.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; abr: " + usageNetworkStats.getAggBytesReceived() + + "; abs: " + usageNetworkStats.getAggBytesSent()); } - currentAccountedBytesSent = (usageNetworkStats.getCurrentBytesSent() + usageNetworkStats.getNetBytesSent()); - currentAccountedBytesReceived = (usageNetworkStats.getCurrentBytesReceived() + usageNetworkStats.getNetBytesReceived()); + currentAccountedBytesSent = usageNetworkStats.getAggBytesSent(); + currentAccountedBytesReceived = usageNetworkStats.getAggBytesReceived(); } - long bytesSent = (userStat.getCurrentBytesSent() + userStat.getNetBytesSent()) - currentAccountedBytesSent; - long bytesReceived = (userStat.getCurrentBytesReceived() + userStat.getNetBytesReceived()) - currentAccountedBytesReceived; + long bytesSent = userStat.getAggBytesSent() - currentAccountedBytesSent; + long bytesReceived = userStat.getAggBytesReceived() - currentAccountedBytesReceived; if (bytesSent < 0) { - s_logger.warn("Calculated negative value for bytes sent: " + bytesSent + ", user stats say: " + (userStat.getCurrentBytesSent() + userStat.getNetBytesSent()) + ", previous network usage was: " + currentAccountedBytesSent); + s_logger.warn("Calculated negative value for bytes sent: " + bytesSent + ", user stats say: " + userStat.getAggBytesSent() + ", previous network usage was: " + currentAccountedBytesSent); bytesSent = 0; } if (bytesReceived < 0) { - s_logger.warn("Calculated negative value for bytes received: " + bytesReceived + ", user stats say: " + (userStat.getCurrentBytesReceived() + userStat.getNetBytesReceived()) + ", previous network usage was: " + currentAccountedBytesReceived); + s_logger.warn("Calculated negative value for bytes received: " + bytesReceived + ", user stats say: " + userStat.getAggBytesReceived() + ", previous network usage was: " + currentAccountedBytesReceived); bytesReceived = 0; } @@ -977,11 +972,10 @@ public class UsageManagerImpl implements UsageManager, Runnable { } UsageNetworkVO usageNetworkVO = new UsageNetworkVO(userStat.getAccountId(), userStat.getDataCenterId(), hostId, userStat.getDeviceType(), userStat.getNetworkId(), bytesSent, bytesReceived, - userStat.getNetBytesReceived(), userStat.getNetBytesSent(), - userStat.getCurrentBytesReceived(), userStat.getCurrentBytesSent(), timestamp); + userStat.getAggBytesReceived(), userStat.getAggBytesSent(), timestamp); if (s_logger.isDebugEnabled()) { - s_logger.debug("creating networkHelperEntry... accountId: " + userStat.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; cbr: " + userStat.getCurrentBytesReceived() + "; cbs: " + userStat.getCurrentBytesSent() + - "; nbr: " + userStat.getNetBytesReceived() + "; nbs: " + userStat.getNetBytesSent() + "; curABS: " + currentAccountedBytesSent + "; curABR: " + currentAccountedBytesReceived + "; ubs: " + bytesSent + "; ubr: " + bytesReceived); + s_logger.debug("creating networkHelperEntry... accountId: " + userStat.getAccountId() + " in zone: " + userStat.getDataCenterId() + "; abr: " + userStat.getAggBytesReceived() + "; abs: " + userStat.getAggBytesSent() + + "; curABS: " + currentAccountedBytesSent + "; curABR: " + currentAccountedBytesReceived + "; ubs: " + bytesSent + "; ubr: " + bytesReceived); } m_usageNetworkDao.persist(usageNetworkVO); }