linstor: improve heartbeat check with also asking linstor (#10105)

If a node doesn't have a DRBD connection to another node,
additionally ask Linstor-Controller if the node is alive.
Otherwise we would have simply said no and the node might still be alive.
This is always the case in a non hyperconverged setup.

plugins/storage/volume/linstor/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to Linstor CloudStack plugin will be documented in this file
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2024-12-13]
+
+### Fixed
+
+- Linstor heartbeat check now also ask linstor-controller if there is no connection between nodes
+
 ## [2024-10-28]
 
 ### Fixed

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java

@@ -45,6 +45,7 @@ import com.linbit.linstor.api.Configuration;
 import com.linbit.linstor.api.DevelopersApi;
 import com.linbit.linstor.api.model.ApiCallRc;
 import com.linbit.linstor.api.model.ApiCallRcList;
+import com.linbit.linstor.api.model.Node;
 import com.linbit.linstor.api.model.Properties;
 import com.linbit.linstor.api.model.ProviderKind;
 import com.linbit.linstor.api.model.Resource;
@@ -712,4 +713,19 @@ public class LinstorStorageAdaptor implements StorageAdaptor {
             throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
         }
     }
+
+    public boolean isNodeOnline(LinstorStoragePool pool, String nodeName) {
+        DevelopersApi linstorApi = getLinstorAPI(pool);
+        try {
+            List<Node> node = linstorApi.nodeList(Collections.singletonList(nodeName), Collections.emptyList(), null, null);
+            if (node == null || node.isEmpty()) {
+                return false;
+            }
+
+            return Node.ConnectionStatusEnum.ONLINE.equals(node.get(0).getConnectionStatus());
+        } catch (ApiException apiEx) {
+            s_logger.error(apiEx.getMessage());
+            throw new CloudRuntimeException(apiEx.getBestMessage(), apiEx);
+        }
+    }
 }

plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStoragePool.java

@@ -279,22 +279,52 @@ public class LinstorStoragePool implements KVMStoragePool {
         return sc.execute(parser);
     }
 
+    private boolean checkLinstorNodeOnline(String nodeName) {
+        return ((LinstorStorageAdaptor)_storageAdaptor).isNodeOnline(this, nodeName);
+    }
+
+    /**
+     * Checks output of drbdsetup status output if this node has any valid connection to the specified
+     * otherNodeName.
+     * If there is no connection, ask the Linstor controller if the node is seen online and return false if not.
+     * If there is a connection but not connected(valid) return false.
+     * @param output Output of the drbdsetup status --json command
+     * @param otherNodeName Name of the node to check against
+     * @return true if we could say that this node thinks the node in question is reachable, otherwise false.
+     */
     private boolean checkDrbdSetupStatusOutput(String output, String otherNodeName) {
         JsonParser jsonParser = new JsonParser();
         JsonArray jResources = (JsonArray) jsonParser.parse(output);
+        boolean connectionFound = false;
         for (JsonElement jElem : jResources) {
             JsonObject jRes = (JsonObject) jElem;
             JsonArray jConnections = jRes.getAsJsonArray("connections");
             for (JsonElement jConElem : jConnections) {
                 JsonObject jConn = (JsonObject) jConElem;
-                if (jConn.getAsJsonPrimitive("name").getAsString().equals(otherNodeName)
+                if (jConn.getAsJsonPrimitive("name").getAsString().equals(otherNodeName))
-                        && jConn.getAsJsonPrimitive("connection-state").getAsString().equalsIgnoreCase("Connected")) {
+                {
-                    return true;
+                    connectionFound = true;
+                    if (jConn.getAsJsonPrimitive("connection-state").getAsString()
+                            .equalsIgnoreCase("Connected")) {
+                        return true;
+                    }
                 }
             }
         }
-        s_logger.warn(String.format("checkDrbdSetupStatusOutput: no resource connected to %s.", otherNodeName));
+        boolean otherNodeOnline = false;
-        return false;
+        if (connectionFound) {
+            s_logger.warn(String.format(
+                    "checkingHeartBeat: connection found, but not in state 'Connected' to %s", otherNodeName));
+        } else {
+            s_logger.warn(String.format(
+                    "checkingHeartBeat: no resource connected to %s, checking LINSTOR", otherNodeName));
+            otherNodeOnline = checkLinstorNodeOnline(otherNodeName);
+        }
+        s_logger.info(String.format(
+                "checkingHeartBeat: other node %s is %s.",
+                otherNodeName,
+                otherNodeOnline ? "online on controller" : "down"));
+        return otherNodeOnline;
     }
 
     private String executeDrbdEventsNow(OutputInterpreter.AllLinesParser parser) {