From 30238f0c7292dc38498f82d146ce147bb7aba72f Mon Sep 17 00:00:00 2001 From: Anshul Gangwar Date: Wed, 13 Nov 2013 17:17:55 +0530 Subject: [PATCH] CLOUDSTACK-4959: added the connection and socket timeout parameters for SMTP and sending message in new thread so that HA doesn't get blocked beacause of hang in sending email alert --- .../src/com/cloud/alert/AlertManagerImpl.java | 49 ++++++++++++++++--- .../src/com/cloud/configuration/Config.java | 5 +- setup/db/db/schema-420to421.sql | 2 + 3 files changed, 49 insertions(+), 7 deletions(-) diff --git a/server/src/com/cloud/alert/AlertManagerImpl.java b/server/src/com/cloud/alert/AlertManagerImpl.java index ed0c9fdcb46..fe7990fcb1e 100755 --- a/server/src/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/com/cloud/alert/AlertManagerImpl.java @@ -25,7 +25,8 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Timer; -import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.ejb.Local; import javax.inject.Inject; @@ -33,6 +34,7 @@ import javax.mail.Authenticator; import javax.mail.Message.RecipientType; import javax.mail.MessagingException; import javax.mail.PasswordAuthentication; +import javax.mail.SendFailedException; import javax.mail.Session; import javax.mail.URLName; import javax.mail.internet.InternetAddress; @@ -80,6 +82,7 @@ import com.cloud.resource.ResourceManager; import com.cloud.storage.StorageManager; import com.cloud.utils.NumbersUtil; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.DB; import com.cloud.utils.db.SearchCriteria; @@ -120,6 +123,12 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi private double _localStorageCapacityThreshold = 0.75; Map _capacityTypeThresholdMap = new HashMap(); + private final ExecutorService _executor; + + public AlertManagerImpl() { + _executor = Executors.newCachedThreadPool(new NamedThreadFactory("Email-Alerts-Sender")); + } + @Override public boolean configure(String name, Map params) throws ConfigurationException { Map configs = _configDao.getConfiguration("management-server", params); @@ -139,12 +148,14 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi String smtpPassword = configs.get("alert.smtp.password"); String emailSender = configs.get("alert.email.sender"); String smtpDebugStr = configs.get("alert.smtp.debug"); + int smtpTimeout = NumbersUtil.parseInt(configs.get("alert.smtp.timeout"), 30000); + int smtpConnectionTimeout = NumbersUtil.parseInt(configs.get("alert.smtp.connectiontimeout"), 30000); boolean smtpDebug = false; if (smtpDebugStr != null) { smtpDebug = Boolean.parseBoolean(smtpDebugStr); } - _emailAlert = new EmailAlert(emailAddresses, smtpHost, smtpPort, useAuth, smtpUsername, smtpPassword, emailSender, smtpDebug); + _emailAlert = new EmailAlert(emailAddresses, smtpHost, smtpPort, smtpConnectionTimeout, smtpTimeout, useAuth, smtpUsername, smtpPassword, emailSender, smtpDebug); String publicIPCapacityThreshold = _configDao.getValue(Config.PublicIpCapacityThreshold.key()); String privateIPCapacityThreshold = _configDao.getValue(Config.PrivateIpCapacityThreshold.key()); @@ -719,8 +730,11 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi private final String _smtpUsername; private final String _smtpPassword; private final String _emailSender; + private int _smtpTimeout; + private int _smtpConnectionTimeout; - public EmailAlert(String[] recipientList, String smtpHost, int smtpPort, boolean smtpUseAuth, final String smtpUsername, final String smtpPassword, String emailSender, boolean smtpDebug) { + public EmailAlert(String[] recipientList, String smtpHost, int smtpPort, int smtpConnectionTimeout, int smtpTimeout, boolean smtpUseAuth, final String smtpUsername, + final String smtpPassword, String emailSender, boolean smtpDebug) { if (recipientList != null) { _recipientList = new InternetAddress[recipientList.length]; for (int i = 0; i < recipientList.length; i++) { @@ -738,12 +752,17 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi _smtpUsername = smtpUsername; _smtpPassword = smtpPassword; _emailSender = emailSender; + _smtpTimeout = smtpTimeout; + _smtpConnectionTimeout = smtpConnectionTimeout; if (_smtpHost != null) { Properties smtpProps = new Properties(); smtpProps.put("mail.smtp.host", smtpHost); smtpProps.put("mail.smtp.port", smtpPort); smtpProps.put("mail.smtp.auth", ""+smtpUseAuth); + smtpProps.put("mail.smtp.timeout", _smtpTimeout); + smtpProps.put("mail.smtp.connectiontimeout", _smtpConnectionTimeout); + if (smtpUsername != null) { smtpProps.put("mail.smtp.user", smtpUsername); } @@ -751,6 +770,9 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi smtpProps.put("mail.smtps.host", smtpHost); smtpProps.put("mail.smtps.port", smtpPort); smtpProps.put("mail.smtps.auth", ""+smtpUseAuth); + smtpProps.put("mail.smtps.timeout", _smtpTimeout); + smtpProps.put("mail.smtps.connectiontimeout", _smtpConnectionTimeout); + if (smtpUsername != null) { smtpProps.put("mail.smtps.user", smtpUsername); } @@ -823,12 +845,27 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi } else { smtpTrans = new SMTPTransport(_smtpSession, new URLName("smtp", _smtpHost, _smtpPort, null, _smtpUsername, _smtpPassword)); } - smtpTrans.connect(); - smtpTrans.sendMessage(msg, msg.getAllRecipients()); - smtpTrans.close(); + sendMessage(smtpTrans, msg); } } + private void sendMessage(final SMTPTransport smtpTrans, final SMTPMessage msg) { + _executor.execute(new Runnable() { + @Override + public void run() { + try { + smtpTrans.connect(); + smtpTrans.sendMessage(msg, msg.getAllRecipients()); + smtpTrans.close(); + } catch (SendFailedException e) { + s_logger.error(" Failed to send email alert " + e); + } catch (MessagingException e) { + s_logger.error(" Failed to send email alert " + e); + } + } + }); + } + public void clearAlert(short alertType, long dataCenterId, Long podId) { if (alertType != -1) { AlertVO alert = _alertDao.getLastAlert(alertType, dataCenterId, podId, null); diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index 5bb135a2e41..d3c0c9ed47a 100755 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -48,6 +48,9 @@ public enum Config { AlertSMTPHost("Alert", ManagementServer.class, String.class, "alert.smtp.host", null, "SMTP hostname used for sending out email alerts.", null), AlertSMTPPassword("Secure", ManagementServer.class, String.class, "alert.smtp.password", null, "Password for SMTP authentication (applies only if alert.smtp.useAuth is true).", null), AlertSMTPPort("Alert", ManagementServer.class, Integer.class, "alert.smtp.port", "465", "Port the SMTP server is listening on.", null), + AlertSMTPConnectionTimeout("Alert", ManagementServer.class, Integer.class, "alert.smtp.connectiontimeout", "30000", + "Socket connection timeout value in milliseconds. -1 for infinite timeout.", null), + AlertSMTPTimeout("Alert", ManagementServer.class, Integer.class, "alert.smtp.timeout", "30000", "Socket I/O timeout value in milliseconds. -1 for infinite timeout.", null), AlertSMTPUseAuth("Alert", ManagementServer.class, String.class, "alert.smtp.useAuth", null, "If true, use SMTP authentication when sending emails.", null), AlertSMTPUsername("Alert", ManagementServer.class, String.class, "alert.smtp.username", null, "Username for SMTP authentication (applies only if alert.smtp.useAuth is true).", null), CapacityCheckPeriod("Alert", ManagementServer.class, Integer.class, "capacity.check.period", "300000", "The interval in milliseconds between capacity checks", null), @@ -77,7 +80,7 @@ public enum Config { CreatePrivateTemplateFromSnapshotWait("Storage", UserVmManager.class, Integer.class, "create.private.template.from.snapshot.wait", "10800", "In second, timeout for CreatePrivateTemplateFromSnapshotCommand", null), BackupSnapshotWait( "Storage", StorageManager.class, Integer.class, "backup.snapshot.wait", "21600", "In second, timeout for BackupSnapshotCommand", null), - HAStorageMigration("Storage", ManagementServer.class, Boolean.class, "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA", null), + HAStorageMigration("Storage", ManagementServer.class, Boolean.class, "enable.ha.storage.migration", "true", "Enable/disable storage migration across primary storage during HA", null), // Network NetworkLBHaproxyStatsVisbility("Network", ManagementServer.class, String.class, "network.loadbalancer.haproxy.stats.visibility", "global", "Load Balancer(haproxy) stats visibilty, the value can be one of the following six parameters : global,guest-network,link-local,disabled,all,default", null), diff --git a/setup/db/db/schema-420to421.sql b/setup/db/db/schema-420to421.sql index 1d2848535bf..fe1e0d49100 100644 --- a/setup/db/db/schema-420to421.sql +++ b/setup/db/db/schema-420to421.sql @@ -224,3 +224,5 @@ update `cloud`.`volumes` v, `cloud`.`volume_host_ref` vhr set v.format=vhr.fo INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'baremetal.ipmi.lan.interface', 'default', 'option specified in -I option of impitool. candidates are: open/bmc/lipmi/lan/lanplus/free/imb, see ipmitool man page for details. default valule "default" means using default option of ipmitool'); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'baremetal.ipmi.fail.retry', 'default', "ipmi interface will be temporary out of order after power opertions(e.g. cycle, on), it leads following commands fail immediately. The value specifies retry times before accounting it as real failure"); INSERT IGNORE INTO `cloud`.`configuration` VALUES ('Advanced', 'DEFAULT', 'management-server', 'vmware.hung.wokervm.timeout', '7200', 'Worker VM timeout in seconds'); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Alert", 'DEFAULT', 'management-server', "alert.smtp.connectiontimeout", "30000", "Socket connection timeout value in milliseconds. -1 for infinite timeout."); +INSERT IGNORE INTO `cloud`.`configuration` VALUES ("Alert", 'DEFAULT', 'management-server', "alert.smtp.timeout", "30000", "Socket I/O timeout value in milliseconds. -1 for infinite timeout."); \ No newline at end of file