mirror of https://github.com/apache/cloudstack.git
CLOUDSTACK-680: Feature SNMP Alerts support in CloudStack
Signed-off-by: Anshul Gangwar <anshul.gangwar@citrix.com> Signed-off-by: Sateesh Chodapuneedi <sateesh@apache.org>
This commit is contained in:
Anshul Gangwar
Sateesh Chodapuneedi
parent
e1c72bc028
commit
61754cd987
|
|
@ -65,6 +65,11 @@
|
|||
<artifactId>cloud-plugin-network-nvp</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cloudstack</groupId>
|
||||
<artifactId>cloud-plugin-snmp-alerts</artifactId>
|
||||
<version>${project.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.cloudstack</groupId>
|
||||
<artifactId>cloud-plugin-network-ovs</artifactId>
|
||||
|
|
|
|||
|
|
@ -74,6 +74,20 @@ under the License.
|
|||
<param name="ConversionPattern" value="%-5p [%c{3}] (%t:%x) %m%n"/>
|
||||
</layout>
|
||||
</appender>
|
||||
<!-- ============================== -->
|
||||
<!-- send alert warnings+ as the SNMP trap if it is configured! -->
|
||||
<!-- ============================== -->
|
||||
|
||||
<appender name="SNMP" class="org.apache.cloudstack.alert.snmp.SnmpTrapAppender">
|
||||
<param name="Threshold" value="WARN"/>
|
||||
<param name="SnmpManagerIpAddresses" value=""/>
|
||||
<param name="SnmpManagerPorts" value=""/>
|
||||
<param name="SnmpManagerCommunities" value=""/>
|
||||
<layout class="org.apache.cloudstack.alert.snmp.SnmpEnhancedPatternLayout">
|
||||
<param name="PairDelimiter" value="//"/>
|
||||
<param name="KeyValueDelimiter" value="::"/>
|
||||
</layout>
|
||||
</appender>
|
||||
|
||||
<!-- ============================== -->
|
||||
<!-- Append messages to the console -->
|
||||
|
|
@ -142,6 +156,17 @@ under the License.
|
|||
<appender-ref ref="AWSAPI"/>
|
||||
</logger>
|
||||
|
||||
<!-- ============================== -->
|
||||
<!-- Add or remove these logger for SNMP, this logger is for SNMP alerts plugin -->
|
||||
<!-- ============================== -->
|
||||
|
||||
<logger name="org.apache.cloudstack.alerts" additivity="false">
|
||||
<level value="WARN"/>
|
||||
<appender-ref ref="SYSLOG"/>
|
||||
<appender-ref ref="CONSOLE"/>
|
||||
<appender-ref ref="FILE"/>
|
||||
<appender-ref ref="SNMP"/>
|
||||
</logger>
|
||||
|
||||
<!-- ======================= -->
|
||||
<!-- Setup the Root category -->
|
||||
|
|
|
|||
|
|
@ -27,26 +27,27 @@ public interface AlertManager extends Manager {
|
|||
public static final short ALERT_TYPE_VIRTUAL_NETWORK_PUBLIC_IP = CapacityVO.CAPACITY_TYPE_VIRTUAL_NETWORK_PUBLIC_IP;
|
||||
public static final short ALERT_TYPE_PRIVATE_IP = CapacityVO.CAPACITY_TYPE_PRIVATE_IP;
|
||||
public static final short ALERT_TYPE_SECONDARY_STORAGE = CapacityVO.CAPACITY_TYPE_SECONDARY_STORAGE;
|
||||
public static final short ALERT_TYPE_HOST = 6;
|
||||
public static final short ALERT_TYPE_USERVM = 7;
|
||||
public static final short ALERT_TYPE_DOMAIN_ROUTER = 8;
|
||||
public static final short ALERT_TYPE_CONSOLE_PROXY = 9;
|
||||
public static final short ALERT_TYPE_ROUTING = 10; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_STORAGE_MISC = 11; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_USAGE_SERVER = 12; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_MANAGMENT_NODE = 13; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_DOMAIN_ROUTER_MIGRATE = 14;
|
||||
public static final short ALERT_TYPE_CONSOLE_PROXY_MIGRATE = 15;
|
||||
public static final short ALERT_TYPE_USERVM_MIGRATE = 16;
|
||||
public static final short ALERT_TYPE_VLAN = 17;
|
||||
public static final short ALERT_TYPE_SSVM = 18;
|
||||
public static final short ALERT_TYPE_USAGE_SERVER_RESULT = 19; // Usage job result
|
||||
public static final short ALERT_TYPE_STORAGE_DELETE = 20;
|
||||
public static final short ALERT_TYPE_UPDATE_RESOURCE_COUNT = 21; // Generated when we fail to update the resource count
|
||||
public static final short ALERT_TYPE_USAGE_SANITY_RESULT = 22;
|
||||
public static final short ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP = 23;
|
||||
public static final short ALERT_TYPE_LOCAL_STORAGE = 24;
|
||||
public static final short ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = 25; // Generated when the resource limit exceeds the limit. Currently used for recurring snapshots only
|
||||
public static final short ALERT_TYPE_HOST = 7;
|
||||
public static final short ALERT_TYPE_USERVM = 8;
|
||||
public static final short ALERT_TYPE_DOMAIN_ROUTER = 9;
|
||||
public static final short ALERT_TYPE_CONSOLE_PROXY = 10;
|
||||
public static final short ALERT_TYPE_ROUTING = 11; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_STORAGE_MISC = 12; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_USAGE_SERVER = 13; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_MANAGMENT_NODE = 14; // lost connection to default route (to the gateway)
|
||||
public static final short ALERT_TYPE_DOMAIN_ROUTER_MIGRATE = 15;
|
||||
public static final short ALERT_TYPE_CONSOLE_PROXY_MIGRATE = 16;
|
||||
public static final short ALERT_TYPE_USERVM_MIGRATE = 17;
|
||||
public static final short ALERT_TYPE_VLAN = 18;
|
||||
public static final short ALERT_TYPE_SSVM = 19;
|
||||
public static final short ALERT_TYPE_USAGE_SERVER_RESULT = 20; // Usage job result
|
||||
public static final short ALERT_TYPE_STORAGE_DELETE = 21;
|
||||
public static final short ALERT_TYPE_UPDATE_RESOURCE_COUNT = 22; // Generated when we fail to update the resource
|
||||
// count
|
||||
public static final short ALERT_TYPE_USAGE_SANITY_RESULT = 23;
|
||||
public static final short ALERT_TYPE_DIRECT_ATTACHED_PUBLIC_IP = 24;
|
||||
public static final short ALERT_TYPE_LOCAL_STORAGE = 25;
|
||||
public static final short ALERT_TYPE_RESOURCE_LIMIT_EXCEEDED = 26; // Generated when the resource limit exceeds the limit. Currently used for recurring snapshots only
|
||||
|
||||
|
||||
void clearAlert(short alertType, long dataCenterId, long podId);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one
|
||||
~ or more contributor license agreements. See the NOTICE file
|
||||
~ distributed with this work for additional information
|
||||
~ regarding copyright ownership. The ASF licenses this file
|
||||
~ to you under the Apache License, Version 2.0 (the
|
||||
~ "License"); you may not use this file except in compliance
|
||||
~ with the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing,
|
||||
~ software distributed under the License is distributed on an
|
||||
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
~ KIND, either express or implied. See the License for the
|
||||
~ specific language governing permissions and limitations
|
||||
~ under the License
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>cloudstack-plugins</artifactId>
|
||||
<groupId>org.apache.cloudstack</groupId>
|
||||
<version>4.2.0-SNAPSHOT</version>
|
||||
<relativePath>../../pom.xml</relativePath>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<name>Apache CloudStack Plugin - SNMP Alerts</name>
|
||||
<artifactId>cloud-plugin-snmp-alerts</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.servicemix.bundles</groupId>
|
||||
<artifactId>org.apache.servicemix.bundles.snmp4j</artifactId>
|
||||
<version>2.1.0_1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>log4j</artifactId>
|
||||
<version>${cs.log4j.version}</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
</project>
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
/**
|
||||
* <p/>
|
||||
* IMPORTANT
|
||||
* <p/>
|
||||
* These OIDs are based on <b>CS-ROOT-MIB</b> MIB file. If there is any change in MIB file
|
||||
* then that should be reflected in this file also *
|
||||
* <br/><br/>
|
||||
* suffix 2 due to conflict with SnmpConstants class of snmp4j
|
||||
*/
|
||||
public class CsSnmpConstants {
|
||||
public static final String CLOUDSTACK = "1.3.6.1.4.1.18060.15";
|
||||
|
||||
public static final String OBJECTS_PREFIX = CLOUDSTACK + ".1.1.";
|
||||
|
||||
public static final String TRAPS_PREFIX = CLOUDSTACK + ".1.2.0.";
|
||||
|
||||
public static final String DATA_CENTER_ID = OBJECTS_PREFIX + 1;
|
||||
|
||||
public static final String POD_ID = OBJECTS_PREFIX + 2;
|
||||
|
||||
public static final String CLUSTER_ID = OBJECTS_PREFIX + 3;
|
||||
|
||||
public static final String MESSAGE = OBJECTS_PREFIX + 4;
|
||||
|
||||
public static final String GENERATION_TIME = OBJECTS_PREFIX + 5;
|
||||
}
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
import org.apache.log4j.EnhancedPatternLayout;
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
public class SnmpEnhancedPatternLayout extends EnhancedPatternLayout {
|
||||
private String _pairDelimiter = "//";
|
||||
private String _keyValueDelimiter = "::";
|
||||
|
||||
private static final int LENGTH_OF_STRING_MESSAGE_AND_KEY_VALUE_DELIMITER = 9;
|
||||
private static final int LENGTH_OF_STRING_MESSAGE = 8;
|
||||
|
||||
public String getKeyValueDelimeter() {
|
||||
return _keyValueDelimiter;
|
||||
}
|
||||
|
||||
public void setKeyValueDelimiter(String keyValueDelimiter) {
|
||||
this._keyValueDelimiter = keyValueDelimiter;
|
||||
}
|
||||
|
||||
public String getPairDelimiter() {
|
||||
return _pairDelimiter;
|
||||
}
|
||||
|
||||
public void setPairDelimiter(String pairDelimiter) {
|
||||
this._pairDelimiter = pairDelimiter;
|
||||
}
|
||||
|
||||
public SnmpTrapInfo parseEvent(LoggingEvent event) {
|
||||
SnmpTrapInfo snmpTrapInfo = null;
|
||||
|
||||
final String message = event.getRenderedMessage();
|
||||
if (message.contains("alertType") && message.contains("message")) {
|
||||
snmpTrapInfo = new SnmpTrapInfo();
|
||||
final StringTokenizer messageSplitter = new StringTokenizer(message, _pairDelimiter);
|
||||
while (messageSplitter.hasMoreTokens()) {
|
||||
final String pairToken = messageSplitter.nextToken();
|
||||
final StringTokenizer pairSplitter = new StringTokenizer(pairToken, _keyValueDelimiter);
|
||||
String keyToken;
|
||||
String valueToken;
|
||||
|
||||
if (pairSplitter.hasMoreTokens()) {
|
||||
keyToken = pairSplitter.nextToken().trim();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
||||
if (pairSplitter.hasMoreTokens()) {
|
||||
valueToken = pairSplitter.nextToken().trim();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
||||
if (keyToken.equalsIgnoreCase("alertType") && !valueToken.equalsIgnoreCase("null")) {
|
||||
snmpTrapInfo.setAlertType(Short.parseShort(valueToken));
|
||||
} else if (keyToken.equalsIgnoreCase("dataCenterId") && !valueToken.equalsIgnoreCase("null")) {
|
||||
snmpTrapInfo.setDataCenterId(Long.parseLong(valueToken));
|
||||
} else if (keyToken.equalsIgnoreCase("podId") && !valueToken.equalsIgnoreCase("null")) {
|
||||
snmpTrapInfo.setPodId(Long.parseLong(valueToken));
|
||||
} else if (keyToken.equalsIgnoreCase("clusterId") && !valueToken.equalsIgnoreCase("null")) {
|
||||
snmpTrapInfo.setClusterId(Long.parseLong(valueToken));
|
||||
} else if (keyToken.equalsIgnoreCase("message") && !valueToken.equalsIgnoreCase("null")) {
|
||||
snmpTrapInfo.setMessage(getSnmpMessage(message));
|
||||
}
|
||||
}
|
||||
|
||||
snmpTrapInfo.setGenerationTime(new Date(event.getTimeStamp()));
|
||||
}
|
||||
return snmpTrapInfo;
|
||||
}
|
||||
|
||||
private String getSnmpMessage(String message) {
|
||||
int lastIndexOfKeyValueDelimiter = message.lastIndexOf(_keyValueDelimiter);
|
||||
int lastIndexOfMessageInString = message.lastIndexOf("message");
|
||||
|
||||
if (lastIndexOfKeyValueDelimiter - lastIndexOfMessageInString <=
|
||||
LENGTH_OF_STRING_MESSAGE_AND_KEY_VALUE_DELIMITER) {
|
||||
return message.substring(lastIndexOfKeyValueDelimiter + _keyValueDelimiter.length()).trim();
|
||||
} else if (lastIndexOfMessageInString < lastIndexOfKeyValueDelimiter) {
|
||||
return message.substring(
|
||||
lastIndexOfMessageInString + _keyValueDelimiter.length() + LENGTH_OF_STRING_MESSAGE).trim();
|
||||
}
|
||||
|
||||
return message.substring(message.lastIndexOf("message" + _keyValueDelimiter) +
|
||||
LENGTH_OF_STRING_MESSAGE_AND_KEY_VALUE_DELIMITER).trim();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import org.snmp4j.CommunityTarget;
|
||||
import org.snmp4j.PDU;
|
||||
import org.snmp4j.Snmp;
|
||||
import org.snmp4j.mp.SnmpConstants;
|
||||
import org.snmp4j.smi.OID;
|
||||
import org.snmp4j.smi.OctetString;
|
||||
import org.snmp4j.smi.UdpAddress;
|
||||
import org.snmp4j.smi.UnsignedInteger32;
|
||||
import org.snmp4j.smi.VariableBinding;
|
||||
import org.snmp4j.transport.DefaultUdpTransportMapping;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
public class SnmpHelper {
|
||||
private Snmp _snmp;
|
||||
private CommunityTarget _target;
|
||||
|
||||
public SnmpHelper(String address, String community) {
|
||||
_target = new CommunityTarget();
|
||||
_target.setCommunity(new OctetString(community));
|
||||
_target.setVersion(SnmpConstants.version2c);
|
||||
_target.setAddress(new UdpAddress(address));
|
||||
try {
|
||||
_snmp = new Snmp(new DefaultUdpTransportMapping());
|
||||
} catch (IOException e) {
|
||||
_snmp = null;
|
||||
throw new CloudRuntimeException(" Error in crearting snmp object, " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
public void sendSnmpTrap(SnmpTrapInfo snmpTrapInfo) {
|
||||
try {
|
||||
if (_snmp != null) {
|
||||
_snmp.send(createPDU(snmpTrapInfo), _target, null, null);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new CloudRuntimeException(" Error in sending SNMP Trap, " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private PDU createPDU(SnmpTrapInfo snmpTrapInfo) {
|
||||
PDU trap = new PDU();
|
||||
trap.setType(PDU.TRAP);
|
||||
|
||||
int alertType = snmpTrapInfo.getAlertType() + 1;
|
||||
if (alertType > 0) {
|
||||
trap.add(new VariableBinding(SnmpConstants.snmpTrapOID, getOID(CsSnmpConstants.TRAPS_PREFIX + alertType)));
|
||||
if (snmpTrapInfo.getDataCenterId() != 0) {
|
||||
trap.add(new VariableBinding(getOID(CsSnmpConstants.DATA_CENTER_ID),
|
||||
new UnsignedInteger32(snmpTrapInfo.getDataCenterId())));
|
||||
}
|
||||
|
||||
if (snmpTrapInfo.getPodId() != 0) {
|
||||
trap.add(new VariableBinding(getOID(CsSnmpConstants.POD_ID), new UnsignedInteger32(snmpTrapInfo
|
||||
.getPodId())));
|
||||
}
|
||||
|
||||
if (snmpTrapInfo.getClusterId() != 0) {
|
||||
trap.add(new VariableBinding(getOID(CsSnmpConstants.CLUSTER_ID), new UnsignedInteger32(snmpTrapInfo
|
||||
.getClusterId())));
|
||||
}
|
||||
|
||||
if (snmpTrapInfo.getMessage() != null) {
|
||||
trap.add(new VariableBinding(getOID(CsSnmpConstants.MESSAGE), new OctetString(snmpTrapInfo.getMessage
|
||||
())));
|
||||
} else {
|
||||
throw new CloudRuntimeException(" What is the use of alert without message ");
|
||||
}
|
||||
|
||||
if (snmpTrapInfo.getGenerationTime() != null) {
|
||||
trap.add(new VariableBinding(getOID(CsSnmpConstants.GENERATION_TIME),
|
||||
new OctetString(snmpTrapInfo.getGenerationTime().toString())));
|
||||
} else {
|
||||
trap.add(new VariableBinding(getOID(CsSnmpConstants.GENERATION_TIME)));
|
||||
}
|
||||
} else {
|
||||
throw new CloudRuntimeException(" Invalid alert Type ");
|
||||
}
|
||||
|
||||
return trap;
|
||||
}
|
||||
|
||||
private OID getOID(String oidString) {
|
||||
return new OID(oidString);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,207 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
import com.cloud.utils.net.NetUtils;
|
||||
import org.apache.log4j.AppenderSkeleton;
|
||||
import org.apache.log4j.spi.ErrorCode;
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.StringTokenizer;
|
||||
|
||||
public class SnmpTrapAppender extends AppenderSkeleton {
|
||||
private String _delimiter = ",";
|
||||
private String _snmpManagerIpAddresses;
|
||||
private String _snmpManagerPorts;
|
||||
private String _snmpManagerCommunities;
|
||||
|
||||
private String _oldSnmpManagerIpAddresses = null;
|
||||
private String _oldSnmpManagerPorts = null;
|
||||
private String _oldSnmpManagerCommunities = null;
|
||||
|
||||
private List<String> _ipAddresses = null;
|
||||
private List<String> _communities = null;
|
||||
private List<String> _ports = null;
|
||||
|
||||
List<SnmpHelper> _snmpHelpers = new ArrayList<SnmpHelper>();
|
||||
|
||||
@Override
|
||||
protected void append(LoggingEvent event) {
|
||||
SnmpEnhancedPatternLayout snmpEnhancedPatternLayout;
|
||||
|
||||
if (getLayout() == null) {
|
||||
errorHandler.error("No layout set for the Appender named [" + getName() + ']', null,
|
||||
ErrorCode.MISSING_LAYOUT);
|
||||
return;
|
||||
}
|
||||
|
||||
if (getLayout() instanceof SnmpEnhancedPatternLayout) {
|
||||
snmpEnhancedPatternLayout = (SnmpEnhancedPatternLayout) getLayout();
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAsSevereAsThreshold(event.getLevel())) {
|
||||
return;
|
||||
}
|
||||
|
||||
SnmpTrapInfo snmpTrapInfo = snmpEnhancedPatternLayout.parseEvent(event);
|
||||
|
||||
if (snmpTrapInfo != null && !_snmpHelpers.isEmpty()) {
|
||||
for (SnmpHelper helper : _snmpHelpers) {
|
||||
try {
|
||||
helper.sendSnmpTrap(snmpTrapInfo);
|
||||
} catch (Exception e) {
|
||||
errorHandler.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void setSnmpHelpers() {
|
||||
if (_snmpManagerIpAddresses == null || _snmpManagerIpAddresses.trim().isEmpty() || _snmpManagerCommunities ==
|
||||
null || _snmpManagerCommunities.trim().isEmpty() || _snmpManagerPorts == null ||
|
||||
_snmpManagerPorts.trim().isEmpty()) {
|
||||
reset();
|
||||
return;
|
||||
}
|
||||
|
||||
if (_oldSnmpManagerIpAddresses != null && _oldSnmpManagerIpAddresses.equals(_snmpManagerIpAddresses) &&
|
||||
_oldSnmpManagerCommunities.equals(_snmpManagerCommunities) &&
|
||||
_oldSnmpManagerPorts.equals(_snmpManagerPorts)) {
|
||||
return;
|
||||
}
|
||||
|
||||
_oldSnmpManagerIpAddresses = _snmpManagerIpAddresses;
|
||||
_oldSnmpManagerPorts = _snmpManagerPorts;
|
||||
_oldSnmpManagerCommunities = _snmpManagerCommunities;
|
||||
|
||||
_ipAddresses = parse(_snmpManagerIpAddresses);
|
||||
_communities = parse(_snmpManagerCommunities);
|
||||
_ports = parse(_snmpManagerPorts);
|
||||
|
||||
if (!(_ipAddresses.size() == _communities.size() && _ipAddresses.size() == _ports.size())) {
|
||||
reset();
|
||||
errorHandler.error(" size of ip addresses , communities, " + "and ports list doesn't match, " +
|
||||
"setting all to null");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!validateIpAddresses() || !validatePorts()) {
|
||||
reset();
|
||||
errorHandler.error(" Invalid format for the IP Addresses or Ports parameter ");
|
||||
return;
|
||||
}
|
||||
|
||||
String address;
|
||||
|
||||
for (int i = 0; i < _ipAddresses.size(); i++) {
|
||||
address = _ipAddresses.get(i) + "/" + _ports.get(i);
|
||||
try {
|
||||
_snmpHelpers.add(new SnmpHelper(address, _communities.get(i)));
|
||||
} catch (Exception e) {
|
||||
errorHandler.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void reset() {
|
||||
_ipAddresses = null;
|
||||
_communities = null;
|
||||
_ports = null;
|
||||
_snmpHelpers.clear();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
if (!closed) closed = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean requiresLayout() {
|
||||
return true;
|
||||
}
|
||||
|
||||
private List<String> parse(String str) {
|
||||
List<String> result = new ArrayList<String>();
|
||||
|
||||
final StringTokenizer tokenizer = new StringTokenizer(str, _delimiter);
|
||||
while (tokenizer.hasMoreTokens()) {
|
||||
result.add(tokenizer.nextToken().trim());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private boolean validatePorts() {
|
||||
for (String port : _ports) {
|
||||
if (!NetUtils.isValidPort(port)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean validateIpAddresses() {
|
||||
for (String ipAddress : _ipAddresses) {
|
||||
if (ipAddress.trim().equalsIgnoreCase("localhost")) {
|
||||
continue;
|
||||
}
|
||||
if (!NetUtils.isValidIp(ipAddress)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public String getSnmpManagerIpAddresses() {
|
||||
return _snmpManagerIpAddresses;
|
||||
}
|
||||
|
||||
public void setSnmpManagerIpAddresses(String snmpManagerIpAddresses) {
|
||||
this._snmpManagerIpAddresses = snmpManagerIpAddresses;
|
||||
setSnmpHelpers();
|
||||
}
|
||||
|
||||
public String getSnmpManagerPorts() {
|
||||
return _snmpManagerPorts;
|
||||
}
|
||||
|
||||
public void setSnmpManagerPorts(String snmpManagerPorts) {
|
||||
this._snmpManagerPorts = snmpManagerPorts;
|
||||
setSnmpHelpers();
|
||||
}
|
||||
|
||||
public String getSnmpManagerCommunities() {
|
||||
return _snmpManagerCommunities;
|
||||
}
|
||||
|
||||
public void setSnmpManagerCommunities(String snmpManagerCommunities) {
|
||||
this._snmpManagerCommunities = snmpManagerCommunities;
|
||||
setSnmpHelpers();
|
||||
}
|
||||
|
||||
public String getDelimiter() {
|
||||
return _delimiter;
|
||||
}
|
||||
|
||||
public void setDelimiter(String delimiter) {
|
||||
this._delimiter = delimiter;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class SnmpTrapInfo {
|
||||
private String message;
|
||||
private long podId;
|
||||
private long dataCenterId;
|
||||
private long clusterId;
|
||||
private Date generationTime;
|
||||
private short alertType;
|
||||
|
||||
public SnmpTrapInfo() {
|
||||
}
|
||||
|
||||
public SnmpTrapInfo(short alertType, long dataCenterId, long podId, long clusterId, String message,
|
||||
Date generationTime) {
|
||||
this.podId = podId;
|
||||
this.alertType = alertType;
|
||||
this.clusterId = clusterId;
|
||||
this.dataCenterId = dataCenterId;
|
||||
this.generationTime = generationTime;
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public short getAlertType() {
|
||||
return alertType;
|
||||
}
|
||||
|
||||
public void setAlertType(short alertType) {
|
||||
this.alertType = alertType;
|
||||
}
|
||||
|
||||
public String getMessage() {
|
||||
return message;
|
||||
}
|
||||
|
||||
public void setMessage(String message) {
|
||||
this.message = message;
|
||||
}
|
||||
|
||||
public long getPodId() {
|
||||
return podId;
|
||||
}
|
||||
|
||||
public void setPodId(long podId) {
|
||||
this.podId = podId;
|
||||
}
|
||||
|
||||
public long getDataCenterId() {
|
||||
return dataCenterId;
|
||||
}
|
||||
|
||||
public void setDataCenterId(long dataCenterId) {
|
||||
this.dataCenterId = dataCenterId;
|
||||
}
|
||||
|
||||
public long getClusterId() {
|
||||
return clusterId;
|
||||
}
|
||||
|
||||
public void setClusterId(long clusterId) {
|
||||
this.clusterId = clusterId;
|
||||
}
|
||||
|
||||
public Date getGenerationTime() {
|
||||
return generationTime;
|
||||
}
|
||||
|
||||
public void setGenerationTime(Date generationTime) {
|
||||
this.generationTime = generationTime;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static junit.framework.Assert.assertNotNull;
|
||||
import static junit.framework.Assert.assertNull;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
public class SnmpEnhancedPatternLayoutTest {
|
||||
SnmpEnhancedPatternLayout _snmpEnhancedPatternLayout = new SnmpEnhancedPatternLayout();
|
||||
|
||||
@Before
|
||||
public void setUp() throws ConfigurationException {
|
||||
_snmpEnhancedPatternLayout.setKeyValueDelimiter("::");
|
||||
_snmpEnhancedPatternLayout.setPairDelimiter("//");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseAlertTest() {
|
||||
LoggingEvent event = mock(LoggingEvent.class);
|
||||
setMessage(" alertType:: 14 // dataCenterId:: 1 // podId:: 1 // " + "clusterId:: null // message:: Management" +
|
||||
" network CIDR is not configured originally. Set it default to 10.102.192.0/22", event);
|
||||
SnmpTrapInfo info = _snmpEnhancedPatternLayout.parseEvent(event);
|
||||
commonAssertions(info, "Management network CIDR is not configured originally. Set it default to 10.102.192" +
|
||||
".0/22");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void ParseAlertWithPairDelimeterInMessageTest() {
|
||||
LoggingEvent event = mock(LoggingEvent.class);
|
||||
setMessage(" alertType:: 14 // dataCenterId:: 1 // podId:: 1 // " + "clusterId:: null // message:: Management" +
|
||||
" //network CIDR is not configured originally. Set it default to 10.102.192.0/22", event);
|
||||
SnmpTrapInfo info = _snmpEnhancedPatternLayout.parseEvent(event);
|
||||
commonAssertions(info, "Management //network CIDR is not configured originally. Set it default to 10.102.192" +
|
||||
".0/22");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void ParseAlertWithKeyValueDelimeterInMessageTest() {
|
||||
LoggingEvent event = mock(LoggingEvent.class);
|
||||
setMessage(" alertType:: 14 // dataCenterId:: 1 // podId:: 1 // " + "clusterId:: null // message:: Management" +
|
||||
" ::network CIDR is not configured originally. Set it default to 10.102.192.0/22", event);
|
||||
SnmpTrapInfo info = _snmpEnhancedPatternLayout.parseEvent(event);
|
||||
commonAssertions(info, "Management ::network CIDR is not configured originally. Set it default to 10.102.192" +
|
||||
".0/22");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseRandomTest() {
|
||||
LoggingEvent event = mock(LoggingEvent.class);
|
||||
when(event.getRenderedMessage()).thenReturn("Problem clearing email alert");
|
||||
assertNull(" Null value was expected ", _snmpEnhancedPatternLayout.parseEvent(event));
|
||||
}
|
||||
|
||||
private void commonAssertions(SnmpTrapInfo info, String message) {
|
||||
assertEquals(" alert type not as expected ", 14, info.getAlertType());
|
||||
assertEquals(" data center id not as expected ", 1, info.getDataCenterId());
|
||||
assertEquals(" pod id os not as expected ", 1, info.getPodId());
|
||||
assertEquals(" cluster id is not as expected ", 0, info.getClusterId());
|
||||
assertNotNull(" generation time is set to null", info.getGenerationTime());
|
||||
assertEquals(" message is not as expected ", message, info.getMessage());
|
||||
}
|
||||
|
||||
private void setMessage(String message, LoggingEvent event) {
|
||||
when(event.getRenderedMessage()).thenReturn(message);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License
|
||||
|
||||
package org.apache.cloudstack.alert.snmp;
|
||||
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mock;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import static junit.framework.Assert.assertEquals;
|
||||
import static junit.framework.Assert.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
|
||||
public class SnmpTrapAppenderTest {
|
||||
SnmpTrapAppender _appender = new SnmpTrapAppender();
|
||||
LoggingEvent _event = mock(LoggingEvent.class);
|
||||
SnmpEnhancedPatternLayout _snmpEnhancedPatternLayout = mock(SnmpEnhancedPatternLayout.class);
|
||||
@Mock
|
||||
List<SnmpHelper> snmpHelpers;
|
||||
|
||||
@Test
|
||||
public void appendTest() {
|
||||
_appender.setSnmpManagerIpAddresses("10.1.1.1,10.1.1.2");
|
||||
_appender.setSnmpManagerPorts("162,164");
|
||||
_appender.setSnmpManagerCommunities("public,snmp");
|
||||
|
||||
_appender.setSnmpHelpers();
|
||||
assertEquals(" error snmpHelper list size not as expected ", _appender._snmpHelpers.size(), 2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void InvalidInputTest() {
|
||||
_appender.setSnmpManagerIpAddresses("10.1.1.1,10.1.1.2");
|
||||
_appender.setSnmpManagerPorts("162,164");
|
||||
_appender.setSnmpManagerCommunities("public");
|
||||
|
||||
_appender.setSnmpHelpers();
|
||||
assertTrue(" list was expected to be empty", _appender._snmpHelpers.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void InvalidIpInputTest() {
|
||||
_appender.setSnmpManagerIpAddresses("10.1.1,10.1.1.2");
|
||||
_appender.setSnmpManagerPorts("162,164");
|
||||
_appender.setSnmpManagerCommunities("public,snmp");
|
||||
|
||||
_appender.setSnmpHelpers();
|
||||
assertTrue(" list was expected to be empty", _appender._snmpHelpers.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void InvalidPortInputTest() {
|
||||
_appender.setSnmpManagerIpAddresses("10.1.1,10.1.1.2");
|
||||
_appender.setSnmpManagerPorts("162,164897489978");
|
||||
_appender.setSnmpManagerCommunities("public,snmp");
|
||||
|
||||
_appender.setSnmpHelpers();
|
||||
assertTrue(" list was expected to be empty", _appender._snmpHelpers.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void mismatchListLengthInputTest() {
|
||||
_appender.setSnmpManagerIpAddresses("10.1.1");
|
||||
_appender.setSnmpManagerPorts("162,164");
|
||||
_appender.setSnmpManagerCommunities("public,snmp");
|
||||
|
||||
_appender.setSnmpHelpers();
|
||||
assertTrue(" list was expected to be empty", _appender._snmpHelpers.isEmpty());
|
||||
}
|
||||
}
|
||||
|
|
@ -57,6 +57,7 @@
|
|||
<module>network-elements/dns-notifier</module>
|
||||
<module>storage/image/s3</module>
|
||||
<module>storage/volume/solidfire</module>
|
||||
<module>alert-handlers/snmp-alerts</module>
|
||||
</modules>
|
||||
|
||||
<dependencies>
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ import com.sun.mail.smtp.SMTPTransport;
|
|||
@Local(value={AlertManager.class})
|
||||
public class AlertManagerImpl extends ManagerBase implements AlertManager {
|
||||
private static final Logger s_logger = Logger.getLogger(AlertManagerImpl.class.getName());
|
||||
private static final Logger s_alertsLogger = Logger.getLogger("org.apache.cloudstack.alerts");
|
||||
|
||||
private static final long INITIAL_CAPACITY_CHECK_DELAY = 30L * 1000L; // thirty seconds expressed in milliseconds
|
||||
|
||||
|
|
@ -256,6 +257,9 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager {
|
|||
try {
|
||||
if (_emailAlert != null) {
|
||||
_emailAlert.sendAlert(alertType, dataCenterId, podId, null, subject, body);
|
||||
} else {
|
||||
s_alertsLogger.warn(" alertType:: " + alertType + " // dataCenterId:: " + dataCenterId + " // podId:: "
|
||||
+ podId + " // clusterId:: " + null + " // message:: " + subject );
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
s_logger.error("Problem sending email alert", ex);
|
||||
|
|
@ -789,6 +793,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager {
|
|||
|
||||
// TODO: make sure this handles SSL transport (useAuth is true) and regular
|
||||
public void sendAlert(short alertType, long dataCenterId, Long podId, Long clusterId, String subject, String content) throws MessagingException, UnsupportedEncodingException {
|
||||
s_alertsLogger.warn(" alertType:: " + alertType + " // dataCenterId:: " + dataCenterId + " // podId:: " +
|
||||
podId + " // clusterId:: " + null + " // message:: " + subject);
|
||||
AlertVO alert = null;
|
||||
if ((alertType != AlertManager.ALERT_TYPE_HOST) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_USERVM) &&
|
||||
|
|
|
|||
|
|
@ -50,11 +50,12 @@ import com.sun.mail.smtp.SMTPTransport;
|
|||
@Local(value={AlertManager.class})
|
||||
public class UsageAlertManagerImpl extends ManagerBase implements AlertManager {
|
||||
private static final Logger s_logger = Logger.getLogger(UsageAlertManagerImpl.class.getName());
|
||||
private static final Logger s_alertsLogger = Logger.getLogger("org.apache.cloudstack.alerts");
|
||||
|
||||
private EmailAlert _emailAlert;
|
||||
@Inject private AlertDao _alertDao;
|
||||
@Inject private ConfigurationDao _configDao;
|
||||
|
||||
|
||||
@Override
|
||||
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
|
||||
Map<String, String> configs = _configDao.getConfiguration("management-server", params);
|
||||
|
|
@ -101,6 +102,9 @@ public class UsageAlertManagerImpl extends ManagerBase implements AlertManager {
|
|||
try {
|
||||
if (_emailAlert != null) {
|
||||
_emailAlert.sendAlert(alertType, dataCenterId, podId, subject, body);
|
||||
} else {
|
||||
s_alertsLogger.warn(" alertType:: " + alertType + " // dataCenterId:: " + dataCenterId + " // podId:: "
|
||||
+ podId + " // clusterId:: " + null + " // message:: " + subject );
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
s_logger.error("Problem sending email alert", ex);
|
||||
|
|
@ -171,18 +175,19 @@ public class UsageAlertManagerImpl extends ManagerBase implements AlertManager {
|
|||
|
||||
// TODO: make sure this handles SSL transport (useAuth is true) and regular
|
||||
public void sendAlert(short alertType, long dataCenterId, Long podId, String subject, String content) throws MessagingException, UnsupportedEncodingException {
|
||||
s_alertsLogger.warn(" alertType:: " + alertType + " // dataCenterId:: " + dataCenterId + " // podId:: " +
|
||||
podId + " // clusterId:: " + null + " // message:: " + subject);
|
||||
AlertVO alert = null;
|
||||
|
||||
if ((alertType != AlertManager.ALERT_TYPE_HOST) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_USERVM) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_DOMAIN_ROUTER) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_CONSOLE_PROXY) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_SSVM) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_SSVM) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_STORAGE_MISC) &&
|
||||
(alertType != AlertManager.ALERT_TYPE_MANAGMENT_NODE)) {
|
||||
alert = _alertDao.getLastAlert(alertType, dataCenterId, podId);
|
||||
}
|
||||
|
||||
|
||||
if (alert == null) {
|
||||
// set up a new alert
|
||||
AlertVO newAlert = new AlertVO();
|
||||
|
|
|
|||
Loading…
Reference in New Issue