diff --git a/console-proxy/scripts/run.sh b/console-proxy/scripts/run.sh index 20de81fac20..4c8aef2dba2 100755 --- a/console-proxy/scripts/run.sh +++ b/console-proxy/scripts/run.sh @@ -26,14 +26,15 @@ while true do - ./_run.sh "$@" + ./_run.sh "$@" & + wait ex=$? if [ $ex -eq 0 ] || [ $ex -eq 1 ] || [ $ex -eq 66 ] || [ $ex -gt 128 ]; then # permanent errors - sleep 160 + sleep 5 elif [ $ex -eq 143 ]; then # service cloud stop causes exit with 143 exit $ex fi - sleep 20 + sleep 5 done diff --git a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyAjaxHandler.java b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyAjaxHandler.java index b1c3f7359cb..bb1f1e21682 100644 --- a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyAjaxHandler.java +++ b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyAjaxHandler.java @@ -285,6 +285,9 @@ public class ConsoleProxyAjaxHandler implements HttpHandler { s_logger.warn("Exception in handle client event bag: " + data + ", ", e); } catch(Exception e) { s_logger.warn("Exception in handle client event bag: " + data + ", ", e); + } catch(OutOfMemoryError e) { + s_logger.error("Unrecoverable OutOfMemory Error, exit and let it be re-launched"); + System.exit(1); } } } diff --git a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyCmdHandler.java b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyCmdHandler.java index 32de7a34eab..27e9be49c55 100644 --- a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyCmdHandler.java +++ b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyCmdHandler.java @@ -45,6 +45,9 @@ public class ConsoleProxyCmdHandler implements HttpHandler { OutputStream os = t.getResponseBody(); os.write(response.getBytes()); os.close(); + } catch(OutOfMemoryError e) { + s_logger.error("Unrecoverable OutOfMemory Error, exit and let it be re-launched"); + System.exit(1); } catch (Throwable e) { s_logger.error(e.toString(), e); } finally { diff --git a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyMonitor.java b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyMonitor.java index 0b87d91f635..45bfd111108 100644 --- a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyMonitor.java +++ b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyMonitor.java @@ -28,6 +28,13 @@ import org.apache.log4j.xml.DOMConfigurator; import com.cloud.console.Logger; +// +// This class is not currently in use, was planning to add a simulated embedded VNC server and monitor console proxy health by +// creating a fake client within the class, but this looks over-complicated and it may not be a reliable approach +// +// I switched to a simpler solution to monitor only unrecoverable exceptions, under these cases, console proxy process will exit +// itself and the shell script will re-launch console proxy +// public class ConsoleProxyMonitor { private static final Logger s_logger = Logger.getLogger(ConsoleProxyMonitor.class); @@ -56,8 +63,8 @@ public class ConsoleProxyMonitor { Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { - onShutdown(); _quit = true; + onShutdown(); } }); @@ -73,10 +80,19 @@ public class ConsoleProxyMonitor { System.exit(1); } - try { - _process.waitFor(); - } catch (InterruptedException e) { - // TODO + boolean waitSucceeded = false; + int exitCode = 0; + while(!waitSucceeded) { + try { + exitCode = _process.waitFor(); + waitSucceeded = true; + + if(s_logger.isInfoEnabled()) + s_logger.info("Console proxy process exits with code: " + exitCode); + } catch (InterruptedException e) { + if(s_logger.isInfoEnabled()) + s_logger.info("InterruptedException while waiting for termination of console proxy, will retry"); + } } } } @@ -97,6 +113,11 @@ public class ConsoleProxyMonitor { } private void onShutdown() { + if(_process != null) { + if(s_logger.isInfoEnabled()) + s_logger.info("Console proxy monitor shuts dwon, terminate console proxy process"); + _process.destroy(); + } } private static void configLog4j() { diff --git a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyThumbnailHandler.java b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyThumbnailHandler.java index 0773c263d9c..690490c3c91 100644 --- a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyThumbnailHandler.java +++ b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyThumbnailHandler.java @@ -61,6 +61,9 @@ public class ConsoleProxyThumbnailHandler implements HttpHandler { OutputStream os = t.getResponseBody(); os.write(response.getBytes()); os.close(); + } catch(OutOfMemoryError e) { + s_logger.error("Unrecoverable OutOfMemory Error, exit and let it be re-launched"); + System.exit(1); } catch (Throwable e) { s_logger.error("Unexpected exception while handing thumbnail request, ", e); diff --git a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyViewer.java b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyViewer.java index 758ae0be791..d3d0153f365 100644 --- a/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyViewer.java +++ b/console-proxy/src/com/cloud/consoleproxy/ConsoleProxyViewer.java @@ -217,6 +217,9 @@ public class ConsoleProxyViewer implements java.lang.Runnable, RfbViewer, RfbPro status = STATUS_AUTHENTICATION_FAILURE; String msg = e.getMessage(); s_logger.warn("Authentication exception, msg: " + msg + "sid: " + this.passwordParam); + } catch(OutOfMemoryError e) { + s_logger.error("Unrecoverable OutOfMemory Error, exit and let it be re-launched"); + System.exit(1); } catch (Exception e) { status = STATUS_ERROR; s_logger.error("Unexpected exception ", e); diff --git a/console/src/com/cloud/console/ConsoleCanvas.java b/console/src/com/cloud/console/ConsoleCanvas.java index fab81977131..f2d5e7e3078 100644 --- a/console/src/com/cloud/console/ConsoleCanvas.java +++ b/console/src/com/cloud/console/ConsoleCanvas.java @@ -358,7 +358,10 @@ public class ConsoleCanvas extends Canvas // rethrow the exception; throw e; - } + } catch(OutOfMemoryError e) { + s_logger.error("Unrecoverable OutOfMemory Error, exit and let it be re-launched"); + System.exit(1); + } } private void processNormalProtocol2() throws Exception { diff --git a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java index 234c69e8e34..aecce0e977c 100644 --- a/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java +++ b/server/src/com/cloud/consoleproxy/ConsoleProxyManagerImpl.java @@ -1457,7 +1457,10 @@ public class ConsoleProxyManagerImpl implements ConsoleProxyManager, ConsoleProx s_logger.error("Unable to send http handling startup command to the console proxy resource for proxy:" + startupCmd.getProxyVmId(), e); } catch (OperationTimedoutException e) { s_logger.error("Unable to send http handling startup command(time out) to the console proxy resource for proxy:" + startupCmd.getProxyVmId(), e); - } catch (Exception e) { + } catch(OutOfMemoryError e) { + s_logger.error("Unrecoverable OutOfMemory Error, exit and let it be re-launched"); + System.exit(1); + } catch (Exception e) { s_logger.error("Unexpected exception when sending http handling startup command(time out) to the console proxy resource for proxy:" + startupCmd.getProxyVmId(), e); } }