Merge pull request #1534 from shapeblue/niotest-fix

CLOUDSTACK-9348: Optimize NioTest and NioConnection main loop- Reduces SSL handshake timeout to 15s, previously this was only 10s in
  commit debfcdef78
- Adds an aggresive explicit wakeup to save the Nio main IO loop/handler from
  getting blocked
- Fix NioTest to fail/succeed in about 60s, previously this was 300s
- Due to aggresive wakeup usage, NioTest should complete in less than 5s on most
  systems. On virtualized environment this may slightly increase due to thread,
  CPU burst/scheduling delays.

/cc @swill  please review and merge.
Sorry about the previous values, they were not optimized for virtualized env. The aggressive selector.wakeup will ensure main IO loop does not get blocked even by malicious users, even for any timeout (ssl handshake etc).

* pr/1534:
  CLOUDSTACK-9348: Optimize NioTest and NioConnection main loop

Signed-off-by: Will Stevens <williamstevens@gmail.com>

utils/src/main/java/com/cloud/utils/nio/Link.java

@@ -596,8 +596,8 @@ public class Link {
         while (handshakeStatus != SSLEngineResult.HandshakeStatus.FINISHED
                 && handshakeStatus != SSLEngineResult.HandshakeStatus.NOT_HANDSHAKING) {
             final long timeTaken = System.currentTimeMillis() - startTimeMills;
-            if (timeTaken > 60000L) {
-                s_logger.warn("SSL Handshake has taken more than 60s to connect to: " + socketChannel.getRemoteAddress() +
+            if (timeTaken > 15000L) {
+                s_logger.warn("SSL Handshake has taken more than 15s to connect to: " + socketChannel.getRemoteAddress() +
                         ". Please investigate this connection.");
                 return false;
             }

utils/src/main/java/com/cloud/utils/nio/NioConnection.java

@@ -171,6 +171,8 @@ public abstract class NioConnection implements Callable<Boolean> {
             } catch (final IOException e) {
                 s_logger.error("Agent will die due to this IOException!", e);
                 throw new NioConnectionException(e.getMessage(), e);
+            } finally {
+                _selector.wakeup();
             }
         }
         _isStartup = false;

utils/src/test/java/com/cloud/utils/testcase/NioTest.java

@@ -61,9 +61,9 @@ public class NioTest {
     private static final Logger LOGGER = Logger.getLogger(NioTest.class);
 
     // Test should fail in due time instead of looping forever
-    private static final int TESTTIMEOUT = 300000;
+    private static final int TESTTIMEOUT = 60000;
 
-    final private int totalTestCount = 5;
+    final private int totalTestCount = 4;
     private int completedTestCount = 0;
 
     private NioServer server;
@@ -71,7 +71,7 @@ public class NioTest {
     private List<NioClient> maliciousClients = new ArrayList<>();
 
     private ExecutorService clientExecutor = Executors.newFixedThreadPool(totalTestCount, new NamedThreadFactory("NioClientHandler"));;
-    private ExecutorService maliciousExecutor = Executors.newFixedThreadPool(5*totalTestCount, new NamedThreadFactory("MaliciousNioClientHandler"));;
+    private ExecutorService maliciousExecutor = Executors.newFixedThreadPool(totalTestCount, new NamedThreadFactory("MaliciousNioClientHandler"));;
 
     private Random randomGenerator = new Random();
     private byte[] testBytes;
@@ -105,12 +105,18 @@ public class NioTest {
             Assert.fail(e.getMessage());
         }
 
+        /**
+         * The malicious client(s) tries to block NioServer's main IO loop
+         * thread until SSL handshake timeout value (from Link class, 15s) after
+         * which the valid NioClient(s) get the opportunity to make connection(s)
+         */
+        for (int i = 0; i < totalTestCount; i++) {
+            final NioClient maliciousClient = new NioMaliciousClient("NioMaliciousTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioMaliciousTestClient());
+            maliciousClients.add(maliciousClient);
+            maliciousExecutor.submit(new ThreadedNioClient(maliciousClient));
+        }
+
         for (int i = 0; i < totalTestCount; i++) {
-            for (int j = 0; j < 4; j++) {
-                final NioClient maliciousClient = new NioMaliciousClient("NioMaliciousTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioMaliciousTestClient());
-                maliciousClients.add(maliciousClient);
-                maliciousExecutor.submit(new ThreadedNioClient(maliciousClient));
-            }
             final NioClient client = new NioClient("NioTestClient-" + i, "127.0.0.1", server.getPort(), 1, new NioTestClient());
             clients.add(client);
             clientExecutor.submit(new ThreadedNioClient(client));
@@ -286,7 +292,6 @@ public class NioTest {
                     LOGGER.info("Server: Received OTHER task");
                 }
             }
-
         }
     }
 }