From 6eb27da5e4d725030662802ebaf89cc7bfef33e3 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Thu, 18 Sep 2025 01:24:04 +1000 Subject: [PATCH 001/145] Fix spelling in Java and Python files; update the ignored words list `codespell.txt` (#11285) Co-authored-by: dahn --- .github/linters/codespell.txt | 3 --- python/lib/cloudutils/utilities.py | 2 +- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 8 ++++---- .../java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java | 8 ++++---- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/linters/codespell.txt b/.github/linters/codespell.txt index 37b3e6de1cb..55918266312 100644 --- a/.github/linters/codespell.txt +++ b/.github/linters/codespell.txt @@ -183,15 +183,12 @@ encryted enebled enmpty entires -enviornment environmnet equivalant erro erronous everthing everytime -excetion -excption excute execept execption diff --git a/python/lib/cloudutils/utilities.py b/python/lib/cloudutils/utilities.py index ce50516193e..5f7e34ed61d 100755 --- a/python/lib/cloudutils/utilities.py +++ b/python/lib/cloudutils/utilities.py @@ -98,7 +98,7 @@ def writeProgressBar(msg, result): sys.stdout.flush() class UnknownSystemException(Exception): - "This Excption is raised if the current operating enviornment is unknown" + "This Exception is raised if the current operating environment is unknown" pass class Distribution: diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 5c5a4044909..eaedaefdae7 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -3113,7 +3113,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous detach volume job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); @@ -3548,7 +3548,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous migrate volume job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); @@ -3895,7 +3895,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous take volume snapshot job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); @@ -4259,7 +4259,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous extract volume job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); diff --git a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index 1fe06d8a3ee..80dbb451530 100644 --- a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -550,7 +550,7 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous create VM snapshot job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); @@ -660,7 +660,7 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous delete VM snapshot job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); @@ -785,7 +785,7 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous revert to snapshot job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); @@ -1036,7 +1036,7 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme } catch (InterruptedException e) { throw new RuntimeException("Operation is interrupted", e); } catch (java.util.concurrent.ExecutionException e) { - throw new RuntimeException("Execution excetion", e); + throw new CloudRuntimeException("Execution exception getting the outcome of the asynchronous delete snapshots job", e); } Object jobResult = _jobMgr.unmarshallResultObject(outcome.getJob()); From 0108ffd1b0063acbd7ca6484f0dc44a88a40e85b Mon Sep 17 00:00:00 2001 From: John Bampton Date: Thu, 18 Sep 2025 02:35:38 +1000 Subject: [PATCH 002/145] java: fix one typo in many files (#11284) --- .../cloud/storage/resource/StorageProcessor.java | 4 ++-- .../StorageSubsystemCommandHandlerBase.java | 6 +++--- ...ckDataStoreStoragePolicyComplianceCommand.java} | 4 ++-- .../allocator/AbstractStoragePoolAllocator.java | 2 +- .../kvm/storage/KVMStorageProcessor.java | 6 +++--- .../ovm3/resources/Ovm3StorageProcessor.java | 6 +++--- .../cloud/resource/SimulatorStorageProcessor.java | 4 ++-- .../vmware/manager/VmwareManagerImpl.java | 4 ++-- .../storage/resource/VmwareStorageProcessor.java | 8 ++++---- .../resource/XenServerStorageProcessor.java | 6 +++--- .../resource/Xenserver625StorageProcessor.java | 6 +++--- .../deploy/DeploymentPlanningManagerImpl.java | 12 ++++++------ .../java/com/cloud/storage/StorageManagerImpl.java | 4 ++-- .../com/cloud/storage/VolumeApiServiceImpl.java | 4 ++-- .../com/cloud/storage/StorageManagerImplTest.java | 14 +++++++------- 15 files changed, 45 insertions(+), 45 deletions(-) rename core/src/main/java/org/apache/cloudstack/storage/command/{CheckDataStoreStoragePolicyComplainceCommand.java => CheckDataStoreStoragePolicyComplianceCommand.java} (93%) diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java index b7cd5b8eb5e..dd8e2abcd64 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java @@ -21,7 +21,7 @@ package com.cloud.storage.resource; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectCommand; import org.apache.cloudstack.storage.command.DeleteCommand; @@ -82,7 +82,7 @@ public interface StorageProcessor { Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd); - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd); + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd); public Answer syncVolumePath(SyncVolumePathCommand cmd); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 58885210120..318c069b0b0 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -21,7 +21,7 @@ package com.cloud.storage.resource; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; @@ -79,8 +79,8 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma return processor.resignature((ResignatureCommand) command); } else if (command instanceof DirectDownloadCommand) { return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command); - } else if (command instanceof CheckDataStoreStoragePolicyComplainceCommand) { - return processor.checkDataStoreStoragePolicyCompliance((CheckDataStoreStoragePolicyComplainceCommand) command); + } else if (command instanceof CheckDataStoreStoragePolicyComplianceCommand) { + return processor.checkDataStoreStoragePolicyCompliance((CheckDataStoreStoragePolicyComplianceCommand) command); } else if (command instanceof SyncVolumePathCommand) { return processor.syncVolumePath((SyncVolumePathCommand) command); } else if (command instanceof QuerySnapshotZoneCopyCommand) { diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplianceCommand.java similarity index 93% rename from core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java rename to core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplianceCommand.java index f9544b873ef..4544e98451f 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplainceCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CheckDataStoreStoragePolicyComplianceCommand.java @@ -21,12 +21,12 @@ package org.apache.cloudstack.storage.command; import com.cloud.agent.api.to.StorageFilerTO; -public class CheckDataStoreStoragePolicyComplainceCommand extends StorageSubSystemCommand { +public class CheckDataStoreStoragePolicyComplianceCommand extends StorageSubSystemCommand { String storagePolicyId; private StorageFilerTO storagePool; - public CheckDataStoreStoragePolicyComplainceCommand(String storagePolicyId, StorageFilerTO storagePool) { + public CheckDataStoreStoragePolicyComplianceCommand(String storagePolicyId, StorageFilerTO storagePool) { super(); this.storagePolicyId = storagePolicyId; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index 4b259760915..e0e244d53d3 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -378,7 +378,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement return false; } } catch (StorageUnavailableException e) { - logger.warn(String.format("Could not verify storage policy complaince against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + logger.warn(String.format("Could not verify storage policy compliance against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); return false; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 3d95157d45c..6aa5b0c0931 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -62,7 +62,7 @@ import org.apache.cloudstack.direct.download.DirectTemplateDownloader; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -3116,8 +3116,8 @@ public class KVMStorageProcessor implements StorageProcessor { } @Override - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { - logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not currently applicable for KVMStorageProcessor"); + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd) { + logger.info("'CheckDataStoreStoragePolicyComplianceCommand' not currently applicable for KVMStorageProcessor"); return new Answer(cmd,false,"Not currently applicable for KVMStorageProcessor"); } diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java index d7c2c214f22..800df1db91f 100644 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java @@ -24,7 +24,7 @@ import java.util.UUID; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -830,8 +830,8 @@ public class Ovm3StorageProcessor implements StorageProcessor { } @Override - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { - logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for Ovm3StorageProcessor"); + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd) { + logger.info("'CheckDataStoreStoragePolicyComplianceCommand' not applicable used for Ovm3StorageProcessor"); return new Answer(cmd,false,"Not applicable used for Ovm3StorageProcessor"); } diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java index 5e0ee17bb8a..3614154f334 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java @@ -32,7 +32,7 @@ import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.command.CreateObjectCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.DettachAnswer; import org.apache.cloudstack.storage.command.DettachCommand; @@ -274,7 +274,7 @@ public class SimulatorStorageProcessor implements StorageProcessor { } @Override - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd) { return new Answer(cmd, true, null); } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 669381884c5..d58af62d5cc 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -60,7 +60,7 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl; import org.apache.cloudstack.management.ManagementServerHost; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; @@ -1574,7 +1574,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw continue; } Collections.shuffle(hostIds); - CheckDataStoreStoragePolicyComplainceCommand command = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicy.getPolicyId(), storageFilerTO); + CheckDataStoreStoragePolicyComplianceCommand command = new CheckDataStoreStoragePolicyComplianceCommand(storagePolicy.getPolicyId(), storageFilerTO); long targetHostId = hypervisorGuruManager.getGuruProcessedCommandTargetHost(hostIds.get(0), command); try { Answer answer = _agentMgr.send(targetHostId, command); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index e4fd5d24f43..af44d9d1476 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -37,7 +37,7 @@ import java.util.concurrent.TimeUnit; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -3776,7 +3776,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } @Override - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd) { String primaryStorageNameLabel = cmd.getStoragePool().getUuid(); String storagePolicyId = cmd.getStoragePolicyId(); VmwareContext context = hostService.getServiceContext(cmd); @@ -3792,14 +3792,14 @@ public class VmwareStorageProcessor implements StorageProcessor { DatastoreMO primaryDsMo = new DatastoreMO(hyperHost.getContext(), morPrimaryDs); boolean isDatastoreStoragePolicyComplaint = primaryDsMo.isDatastoreStoragePolicyComplaint(storagePolicyId); - String failedMessage = String.format("DataStore %s is not complaince with storage policy id %s", primaryStorageNameLabel, storagePolicyId); + String failedMessage = String.format("DataStore %s is not compliance with storage policy id %s", primaryStorageNameLabel, storagePolicyId); if (!isDatastoreStoragePolicyComplaint) return new Answer(cmd, isDatastoreStoragePolicyComplaint, failedMessage); else return new Answer(cmd, isDatastoreStoragePolicyComplaint, null); } catch (Throwable e) { hostService.createLogMessageException(e, cmd); - String details = String.format("Exception while checking if datastore [%s] is storage policy [%s] complaince due to: [%s]", primaryStorageNameLabel, storagePolicyId, VmwareHelper.getExceptionMessage(e)); + String details = String.format("Exception while checking if datastore [%s] is storage policy [%s] compliance due to: [%s]", primaryStorageNameLabel, storagePolicyId, VmwareHelper.getExceptionMessage(e)); return new Answer(cmd, false, details); } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java index a214131c47d..55cae3aaa0f 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java @@ -36,7 +36,7 @@ import java.util.UUID; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.storage.command.AttachAnswer; import org.apache.cloudstack.storage.command.AttachCommand; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.CreateObjectAnswer; @@ -221,8 +221,8 @@ public class XenServerStorageProcessor implements StorageProcessor { } @Override - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { - logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd) { + logger.info("'CheckDataStoreStoragePolicyComplianceCommand' not applicable used for XenServerStorageProcessor"); return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java index c7b288ef8c7..3ecbafc0926 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java @@ -27,7 +27,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; @@ -919,8 +919,8 @@ public class Xenserver625StorageProcessor extends XenServerStorageProcessor { } @Override - public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { - logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplianceCommand cmd) { + logger.info("'CheckDataStoreStoragePolicyComplianceCommand' not applicable used for XenServerStorageProcessor"); return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); } diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index daaebb42a34..bb165e0529d 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -1538,12 +1538,12 @@ StateListener, Configurable { hostCanAccessPool = true; if (potentialHost.getHypervisorType() == HypervisorType.VMware) { try { - boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolCompliantWithStoragePolicy(volumeDiskProfilePair, storagePool); - if (!isStoragePoolStoragepolicyComplaince) { + boolean isStoragePoolStoragepolicyCompliance = _storageMgr.isStoragePoolCompliantWithStoragePolicy(volumeDiskProfilePair, storagePool); + if (!isStoragePoolStoragepolicyCompliance) { continue; } } catch (StorageUnavailableException e) { - logger.warn("Could not verify storage policy complaince against storage pool {} due to exception {}", storagePool, e.getMessage()); + logger.warn("Could not verify storage policy compliance against storage pool {} due to exception {}", storagePool, e.getMessage()); continue; } haveEnoughSpace = true; @@ -1577,12 +1577,12 @@ StateListener, Configurable { List> volumeDiskProfilePair = getVolumeDiskProfilePairs(requestVolumes); if (potentialHost.getHypervisorType() == HypervisorType.VMware) { try { - boolean isStoragePoolStoragepolicyComplaince = _storageMgr.isStoragePoolCompliantWithStoragePolicy(volumeDiskProfilePair, potentialSPool); - if (!isStoragePoolStoragepolicyComplaince) { + boolean isStoragePoolStoragepolicyCompliance = _storageMgr.isStoragePoolCompliantWithStoragePolicy(volumeDiskProfilePair, potentialSPool); + if (!isStoragePoolStoragepolicyCompliance) { continue; } } catch (StorageUnavailableException e) { - logger.warn("Could not verify storage policy complaince against storage pool {} due to exception {}", potentialSPool, e.getMessage()); + logger.warn("Could not verify storage policy compliance against storage pool {} due to exception {}", potentialSPool, e.getMessage()); continue; } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index e1833397e99..cbeec1a60b3 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -122,7 +122,7 @@ import org.apache.cloudstack.secstorage.HeuristicVO; import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.secstorage.heuristics.HeuristicType; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; @@ -3614,7 +3614,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } try { StorageFilerTO storageFilerTO = new StorageFilerTO(pool); - CheckDataStoreStoragePolicyComplainceCommand cmd = new CheckDataStoreStoragePolicyComplainceCommand(storagePolicyVO.getPolicyId(), storageFilerTO); + CheckDataStoreStoragePolicyComplianceCommand cmd = new CheckDataStoreStoragePolicyComplianceCommand(storagePolicyVO.getPolicyId(), storageFilerTO); long targetHostId = _hvGuruMgr.getGuruProcessedCommandTargetHost(hostIds.get(0), cmd); return _agentMgr.send(targetHostId, cmd); } catch (AgentUnavailableException e) { diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index eaedaefdae7..f4536e8549f 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -3507,8 +3507,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic if (hypervisorType.equals(HypervisorType.VMware)) { try { - boolean isStoragePoolStoragepolicyComplaince = storageMgr.isStoragePoolCompliantWithStoragePolicy(Arrays.asList(volumeDiskProfilePair), destPool); - if (!isStoragePoolStoragepolicyComplaince) { + boolean isStoragePoolStoragepolicyCompliance = storageMgr.isStoragePoolCompliantWithStoragePolicy(Arrays.asList(volumeDiskProfilePair), destPool); + if (!isStoragePoolStoragepolicyCompliance) { throw new CloudRuntimeException(String.format("Storage pool %s is not storage policy compliance with the volume %s", poolUuid, vol.getUuid())); } } catch (StorageUnavailableException e) { diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index 6eb8bd04f46..576c32c08ba 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -43,7 +43,7 @@ import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; -import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; +import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplianceCommand; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -453,7 +453,7 @@ public class StorageManagerImplTest { } try { Mockito.doReturn(new com.cloud.agent.api.Answer( - Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class))) + Mockito.mock(CheckDataStoreStoragePolicyComplianceCommand.class))) .when(storageManagerImpl).getCheckDatastorePolicyComplianceAnswer("policy", pool); assertTrue(storageManagerImpl.isStoragePoolCompliantWithStoragePolicy(1L, pool)); } catch (StorageUnavailableException e) { @@ -461,7 +461,7 @@ public class StorageManagerImplTest { } try { com.cloud.agent.api.Answer answer = - new com.cloud.agent.api.Answer(Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class), + new com.cloud.agent.api.Answer(Mockito.mock(CheckDataStoreStoragePolicyComplianceCommand.class), false, ""); Mockito.doReturn(answer) .when(storageManagerImpl).getCheckDatastorePolicyComplianceAnswer("policy", pool); @@ -502,7 +502,7 @@ public class StorageManagerImplTest { .thenReturn(policy); Mockito.doReturn(new ArrayList<>(List.of(1L, 2L))).when(storageManagerImpl).getUpHostsInPool(Mockito.anyLong()); Mockito.when(hvGuruMgr.getGuruProcessedCommandTargetHost(Mockito.anyLong(), - Mockito.any(CheckDataStoreStoragePolicyComplainceCommand.class))).thenReturn(1L); + Mockito.any(CheckDataStoreStoragePolicyComplianceCommand.class))).thenReturn(1L); try { Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(Command.class))) .thenThrow(AgentUnavailableException.class); @@ -529,12 +529,12 @@ public class StorageManagerImplTest { .thenReturn(policy); Mockito.doReturn(new ArrayList<>(List.of(1L, 2L))).when(storageManagerImpl).getUpHostsInPool(Mockito.anyLong()); Mockito.when(hvGuruMgr.getGuruProcessedCommandTargetHost(Mockito.anyLong(), - Mockito.any(CheckDataStoreStoragePolicyComplainceCommand.class))).thenReturn(1L); + Mockito.any(CheckDataStoreStoragePolicyComplianceCommand.class))).thenReturn(1L); try { Mockito.when(agentManager.send(Mockito.anyLong(), - Mockito.any(CheckDataStoreStoragePolicyComplainceCommand.class))) + Mockito.any(CheckDataStoreStoragePolicyComplianceCommand.class))) .thenReturn(new com.cloud.agent.api.Answer( - Mockito.mock(CheckDataStoreStoragePolicyComplainceCommand.class))); + Mockito.mock(CheckDataStoreStoragePolicyComplianceCommand.class))); } catch (AgentUnavailableException | OperationTimedoutException e) { Assert.fail(e.getMessage()); } From 0526ec0d82a7f08eab314921b322b1ef4c364a34 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Thu, 18 Sep 2025 18:01:24 +1000 Subject: [PATCH 003/145] PULL_REQUEST_TEMPLATE standardize case of types of changes (#11663) --- PULL_REQUEST_TEMPLATE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 0bd7a7643ce..be04436d3ff 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -22,8 +22,8 @@ This PR... - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] Enhancement (improves an existing feature and functionality) - [ ] Cleanup (Code refactoring and cleanup, that may add test cases) -- [ ] build/CI -- [ ] test (unit or integration test code) +- [ ] Build/CI +- [ ] Test (unit or integration test code) ### Feature/Enhancement Scale or Bug Severity From 8eb6ddd920311cc152b591a2e1e16a7598656195 Mon Sep 17 00:00:00 2001 From: Pearl Dsilva Date: Thu, 18 Sep 2025 07:25:48 -0400 Subject: [PATCH 004/145] Fix inactive VPCs deletion (#11618) Co-authored-by: Nicolas Vazquez --- .../cloudstack/service/NetrisApiClientImpl.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisApiClientImpl.java b/plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisApiClientImpl.java index 6b047f8df4b..9d3ff953ee1 100644 --- a/plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisApiClientImpl.java +++ b/plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisApiClientImpl.java @@ -1131,16 +1131,17 @@ public class NetrisApiClientImpl implements NetrisApiClient { public boolean deleteVpc(DeleteNetrisVpcCommand cmd) { String suffix = String.valueOf(cmd.getId()); String vpcName = NetrisResourceObjectUtils.retrieveNetrisResourceObjectName(cmd, NetrisResourceObjectUtils.NetrisObjectType.VPC); - VPCListing vpcResource = getVpcByNameAndTenant(vpcName); - if (vpcResource == null) { - logger.error("Could not find the Netris VPC resource with name {} and tenant ID {}", vpcName, tenantId); - return false; - } String snatRuleName = NetrisResourceObjectUtils.retrieveNetrisResourceObjectName(cmd, NetrisResourceObjectUtils.NetrisObjectType.SNAT, suffix); NatGetBody existingNatRule = netrisNatRuleExists(snatRuleName); boolean ruleExists = Objects.nonNull(existingNatRule); if (ruleExists) { - deleteNatRule(snatRuleName, existingNatRule.getId(), vpcResource.getName()); + deleteNatRule(snatRuleName, existingNatRule.getId(), vpcName); + } + + VPCListing vpcResource = getVpcByNameAndTenant(vpcName); + if (vpcResource == null) { + logger.warn("The Netris VPC resource with name {} and tenant ID {} does not exist, cannot be removed", vpcName, tenantId); + return true; } String vpcAllocationName = NetrisResourceObjectUtils.retrieveNetrisResourceObjectName(cmd, NetrisResourceObjectUtils.NetrisObjectType.IPAM_ALLOCATION, cmd.getCidr()); From 9167cd3b72253cb509b19804f3e740b90b41a7af Mon Sep 17 00:00:00 2001 From: Vishesh Date: Fri, 19 Sep 2025 17:38:12 +0530 Subject: [PATCH 005/145] server: use /prod/stat to get uptime instead of the uptime command (#11670) --- server/src/main/java/com/cloud/server/StatsCollector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index a32dac398a8..4e1e78f1610 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -937,12 +937,12 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc logger.info(String.format("used memory from /proc: %d", newEntry.getSystemMemoryUsed())); } try { - String bootTime = Script.runSimpleBashScript("uptime -s"); - SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss", Locale.ENGLISH); + String bootTime = Script.runSimpleBashScript("date -d @$(grep btime /proc/stat | awk '{print $2}') '+%Y-%m-%d %H:%M:%S'"); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH); Date date = formatter.parse(bootTime); newEntry.setSystemBootTime(date); } catch (ParseException e) { - logger.error("can not retrieve system uptime"); + logger.error("can not retrieve system uptime", e); } String maxuse = Script.runSimpleBashScript(String.format("ps -o vsz= %d", newEntry.getPid())); newEntry.setSystemMemoryVirtualSize(Long.parseLong(maxuse) * 1024); From a452dc6543f6fdb5242d46e07167b7c1450706b6 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Fri, 19 Sep 2025 23:20:40 +1000 Subject: [PATCH 006/145] chore(markdown): use https on links (#11676) --- CHANGES.md | 4 ++-- INSTALL.md | 6 +++--- README.md | 10 +++++----- tools/devcloud4/advanced/README.md | 2 +- tools/devcloud4/basic/README.md | 2 +- ui/docs/full-test-plan.template.md | 2 +- ui/docs/smoke-test-plan.template.md | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8fea4f3a355..7a670a55a61 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -644,12 +644,12 @@ Bug ID | Description Version 4.2.1 ------------- -Release notes contain the list of [bug fixes](http://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.1/html/Release_Notes/version-4.2.html#issues-fixed-4.2.1) +Release notes contain the list of [bug fixes](https://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.1/html/Release_Notes/version-4.2.html#issues-fixed-4.2.1) Version 4.2.0 ------------- Released on October 1 2013. -Release notes contain the list of [bug fixes](http://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.0/html/Release_Notes/index.html) +Release notes contain the list of [bug fixes](https://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.0/html/Release_Notes/index.html) Version 4.1.0 ------------- diff --git a/INSTALL.md b/INSTALL.md index efb42678b0a..b9dfd908efc 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,7 +1,7 @@ # Apache CloudStack Installation basics This document describes how to develop, build, package and install Apache -CloudStack. For more information please refer to the official [documentation](http://docs.cloudstack.apache.org) +CloudStack. For more information please refer to the official [documentation](https://docs.cloudstack.apache.org) or the developer [wiki](https://cwiki.apache.org/confluence/display/CLOUDSTACK/Home). Apache CloudStack developers use various platforms for development, this guide @@ -41,7 +41,7 @@ Start the MySQL service: ### Using jenv and/or pyenv for Version Management -CloudStack is built using Java and Python. To make selection of these tools versions more consistent and ease installation for developers, optional support for [jenv](http://www.jenv.be/) and [pyenv](https://github.com/yyuu/pyenv) with [virtualenv]|(https://github.com/yyuu/pyenv-virtualenv) is provided. jenv installation instructions are available here and pyenv installation instructions are available here. For users of [oh-my-zsh](http://ohmyz.sh/) there is a pyenv plugin available to trigger configuration of pyenv in a shell session. +CloudStack is built using Java and Python. To make selection of these tools versions more consistent and ease installation for developers, optional support for [jenv](http://www.jenv.be/) and [pyenv](https://github.com/yyuu/pyenv) with [virtualenv]|(https://github.com/yyuu/pyenv-virtualenv) is provided. jenv installation instructions are available here and pyenv installation instructions are available here. For users of [oh-my-zsh](https://ohmyz.sh/) there is a pyenv plugin available to trigger configuration of pyenv in a shell session. Following installation, execute the following commands to configure jenv and pyenv for use with CloudStack development: @@ -171,7 +171,7 @@ All the rpm packages will be created in `dist/rpmbuild/RPMS/x86_64` directory. ## Notes -If you will be using Xen as your hypervisor, please download [vhd-util](http://download.cloudstack.org/tools/vhd-util) +If you will be using Xen as your hypervisor, please download [vhd-util](https://download.cloudstack.org/tools/vhd-util) If management server is installed on RHEL/CentOS, then copy vhd-util into: diff --git a/README.md b/README.md index 2403b20d652..08540a0628d 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ OVM and LXC containers. Users can manage their cloud with an easy to use Web interface, command line tools, and/or a full-featured query based API. -For more information on Apache CloudStack, please visit the [website](http://cloudstack.apache.org) +For more information on Apache CloudStack, please visit the [website](https://cloudstack.apache.org) ## Who Uses CloudStack? @@ -88,10 +88,10 @@ via GitHub pull requests. ## Getting Involved and Contributing Interested in helping out with Apache CloudStack? Great! We welcome -participation from anybody willing to work [The Apache Way](http://theapacheway.com) and make a +participation from anybody willing to work [The Apache Way](https://theapacheway.com) and make a contribution. Note that you do not have to be a developer in order to contribute to Apache CloudStack. We need folks to help with documentation, translation, -promotion etc. See our contribution [page](http://cloudstack.apache.org/contribute.html). +promotion etc. See our contribution [page](https://cloudstack.apache.org/contribute.html). If you are a frequent contributors, you can request to be added as collaborators (see https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features#Git.asf.yamlfeatures-AssigningexternalcollaboratorswiththetriageroleonGitHub) @@ -102,7 +102,7 @@ You may do so by sharing your GitHub users ID or raise a GitHub issue. If you're interested in learning more or participating in the Apache CloudStack project, the mailing lists are the best way to do that. While the project has -several communications channels, the [mailing lists](http://cloudstack.apache.org/mailing-lists.html) are the most active and the +several communications channels, the [mailing lists](https://cloudstack.apache.org/mailing-lists.html) are the most active and the official channels for making decisions about the project itself. Mailing lists: @@ -122,7 +122,7 @@ released version of CloudStack, please report it to `security@apache.org` with details about the vulnerability, how it might be exploited, and any additional information that might be useful. -For more details, please visit our security [page](http://cloudstack.apache.org/security.html). +For more details, please visit our security [page](https://cloudstack.apache.org/security.html). ## License diff --git a/tools/devcloud4/advanced/README.md b/tools/devcloud4/advanced/README.md index 986653fbc3b..8c82b4e3a50 100644 --- a/tools/devcloud4/advanced/README.md +++ b/tools/devcloud4/advanced/README.md @@ -52,7 +52,7 @@ vagrant up ```bash cd /path/to/cloudstack/repo - wget http://download.cloudstack.org/tools/vhd-util -P scripts/vm/hypervisor/xenserver/ + wget https://download.cloudstack.org/tools/vhd-util -P scripts/vm/hypervisor/xenserver/ chmod +x scripts/vm/hypervisor/xenserver/vhd-util ``` diff --git a/tools/devcloud4/basic/README.md b/tools/devcloud4/basic/README.md index b0bfdaaa412..21b5d97fd4a 100644 --- a/tools/devcloud4/basic/README.md +++ b/tools/devcloud4/basic/README.md @@ -42,7 +42,7 @@ vagrant up ```bash cd /path/to/cloudstack/repo - wget http://download.cloudstack.org/tools/vhd-util -P scripts/vm/hypervisor/xenserver/ + wget https://download.cloudstack.org/tools/vhd-util -P scripts/vm/hypervisor/xenserver/ chmod +x scripts/vm/hypervisor/xenserver/vhd-util ``` diff --git a/ui/docs/full-test-plan.template.md b/ui/docs/full-test-plan.template.md index a93959ad10c..259ec5d9858 100644 --- a/ui/docs/full-test-plan.template.md +++ b/ui/docs/full-test-plan.template.md @@ -33,7 +33,7 @@ Note: for User role test exclude after Account/User feature, for DomainAdmin rol - [ ] Create new instance **Compute > Kubernetes** -This requires configuring and setting up CKS: http://docs.cloudstack.apache.org/en/latest/plugins/cloudstack-kubernetes-service.html +This requires configuring and setting up CKS: https://docs.cloudstack.apache.org/en/latest/plugins/cloudstack-kubernetes-service.html - [ ] Basic search - [ ] Extended search - [ ] Sort diff --git a/ui/docs/smoke-test-plan.template.md b/ui/docs/smoke-test-plan.template.md index b748874335e..ca1e1fbd42e 100644 --- a/ui/docs/smoke-test-plan.template.md +++ b/ui/docs/smoke-test-plan.template.md @@ -17,7 +17,7 @@ Note: for User role test exclude after Account/User feature, for DomainAdmin rol - [ ] Add/delete secondary IP address **Compute > Kubernetes** -This requires configuring and setting up CKS: http://docs.cloudstack.apache.org/en/latest/plugins/cloudstack-kubernetes-service.html +This requires configuring and setting up CKS: https://docs.cloudstack.apache.org/en/latest/plugins/cloudstack-kubernetes-service.html - [ ] Add Kubernetes cluster - [ ] Start/stop a Kubernetes cluster - [ ] Scale Kubernetes cluster From a466750d3b4f141621b96368a8b27a09c1f7d818 Mon Sep 17 00:00:00 2001 From: julien-vaz <54545601+julien-vaz@users.noreply.github.com> Date: Fri, 19 Sep 2025 10:46:20 -0300 Subject: [PATCH 007/145] Add cleanup for tiers dropdown on assignVIrtualMachine API form (#11464) Co-authored-by: Julien Hervot de Mattos Vaz --- ui/src/views/compute/AssignInstance.vue | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/views/compute/AssignInstance.vue b/ui/src/views/compute/AssignInstance.vue index 274c68cd702..9b61125d033 100644 --- a/ui/src/views/compute/AssignInstance.vue +++ b/ui/src/views/compute/AssignInstance.vue @@ -107,6 +107,7 @@ export default { this.fetchNetworks() }, fetchNetworks () { + this.selectedNetwork = null this.loading = true var params = { domainId: this.selectedDomain, From e6c7a71d40e5dd0272df718f9b2231e6f9461876 Mon Sep 17 00:00:00 2001 From: vanzan01 <35181591+vanzan01@users.noreply.github.com> Date: Sat, 20 Sep 2025 01:16:00 +0800 Subject: [PATCH 008/145] Add yamllint pre-commit hook for YAML file standardization (#11017) * Add yamllint pre-commit hook for YAML file standardization Add yamllint hook to .pre-commit-config.yaml to standardize YAML files across the project. Created .yamllint.yml configuration with relaxed rules to accommodate existing files. Fixed colon spacing in .asf.yaml to pass yamllint checks. Closes #10974 Signed-off-by: vanzan01 * Add extra eol to .yamllint.yml Address review comment from DaanHoogland Signed-off-by: vanzan01 * Move .yamllint.yml to .github/linters/ and improve file pattern - Move .yamllint.yml to .github/linters/ directory for better organization - Update pre-commit config to point to new location - Change file pattern to \.ya?ml$ as suggested Address review feedback from jbampton Signed-off-by: vanzan01 * Update .pre-commit-config.yaml Co-authored-by: John Bampton * Make yamllint more forgiving and exclude only k8s templates - Remove extensive file exclusions, keep only k8s templates - Increase line length limit to 400 chars (GitHub Actions friendly) - Allow flexible colon spacing (max-spaces-after: -1) - Disable document-start requirement This provides linting value while handling infrastructure patterns. Signed-off-by: vanzan01 * Update .pre-commit-config.yaml Co-authored-by: John Bampton --------- Signed-off-by: vanzan01 Co-authored-by: John Bampton --- .asf.yaml | 6 +++--- .github/linters/.yamllint.yml | 31 +++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 10 ++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 .github/linters/.yamllint.yml diff --git a/.asf.yaml b/.asf.yaml index be818fda4d3..a8f1482cf4a 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -63,7 +63,7 @@ github: protected_branches: ~ notifications: - commits: commits@cloudstack.apache.org - issues: commits@cloudstack.apache.org + commits: commits@cloudstack.apache.org + issues: commits@cloudstack.apache.org pullrequests: commits@cloudstack.apache.org - discussions: users@cloudstack.apache.org + discussions: users@cloudstack.apache.org diff --git a/.github/linters/.yamllint.yml b/.github/linters/.yamllint.yml new file mode 100644 index 00000000000..8a72aee0adb --- /dev/null +++ b/.github/linters/.yamllint.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +extends: relaxed + +rules: + line-length: + max: 400 # Very forgiving for GitHub Actions and infrastructure files + indentation: disable # Disable indentation checking for existing files + comments: disable # Disable comment formatting checks + brackets: disable # Disable bracket spacing checks + colons: + max-spaces-after: -1 # Allow any number of spaces after colon + max-spaces-before: 0 + document-start: disable # Many files don't have --- + truthy: + allowed-values: ['true', 'false', 'on', 'off', 'yes', 'no'] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46db709c3fe..70d7e39cdf3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -94,3 +94,13 @@ repos: args: [--config=.github/linters/.markdown-lint.yml] types: [markdown] files: \.(md|mdown|markdown)$ + - repo: https://github.com/adrienverge/yamllint + rev: v1.37.1 + hooks: + - id: yamllint + name: run yamllint + description: check YAML files with yamllint + args: [--config-file=.github/linters/.yamllint.yml] + types: [yaml] + files: \.ya?ml$ + exclude: ^.*k8s-.*\.ya?ml$ From 3ef25569000e817433f4348cdd27d39736a33792 Mon Sep 17 00:00:00 2001 From: Harikrishna Date: Sat, 20 Sep 2025 17:57:45 +0530 Subject: [PATCH 009/145] Fix scaleKubernetesCluster API (#11652) * Fix scaleKubernetesCluster * Added more state transitions --- .../java/com/cloud/kubernetes/cluster/KubernetesCluster.java | 2 ++ .../cluster/actionworkers/KubernetesClusterScaleWorker.java | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 571d993c7a1..60966798cbb 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -88,6 +88,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm s_fsm.addTransition(State.Stopping, Event.OperationFailed, State.Alert); s_fsm.addTransition(State.Stopped, Event.StartRequested, State.Starting); + s_fsm.addTransition(State.Stopped, Event.OperationSucceeded, State.Stopped); + s_fsm.addTransition(State.Running, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Running, Event.FaultsDetected, State.Alert); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 0938fd44554..c552b5d57c7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -582,8 +582,6 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif scaleKubernetesClusterOffering(nodeType, scalingServiceOffering, updateNodeOffering, updateClusterOffering); } else if (clusterSizeScalingNeeded) { scaleKubernetesClusterSize(nodeType); - } else { - return true; } } From aca8732102f6033f9ee60f71731665c0a0227a02 Mon Sep 17 00:00:00 2001 From: dahn Date: Mon, 22 Sep 2025 08:09:05 +0200 Subject: [PATCH 010/145] [router] make a distinction between fatal errors, warnings and unknown as healthcheck result (#10710) * [routers] distiction between fatal failure and warning or unknown on healthchecks * UI status for router health checks * status from scripts varied * automation signalled errors * revert removal of update sql * upgradeversion * move config item and further cleanup * handling services better * backwards compatible response --------- Co-authored-by: Daan Hoogland --- .../configuration/ConfigurationService.java | 60 +- .../network/RouterHealthCheckResult.java | 2 +- .../VirtualNetworkApplianceService.java | 4 + .../network/lb/LoadBalancingRulesService.java | 20 +- .../RouterHealthCheckResultResponse.java | 19 +- .../service/NetworkOrchestrationService.java | 51 +- .../configuration/ConfigurationManager.java | 121 +--- .../orchestration/NetworkOrchestrator.java | 135 +++-- .../dao/RouterHealthCheckResultVO.java | 7 +- .../db/schema-42010to42100cleanup.sql | 20 - .../META-INF/db/schema-42100to42200.sql | 4 + .../framework/config/ConfigKey.java | 14 +- .../lb/ElasticLoadBalancerManagerImpl.java | 3 +- .../lb/InternalLoadBalancerVMManagerImpl.java | 2 +- .../java/com/cloud/api/ApiResponseHelper.java | 12 +- .../java/com/cloud/configuration/Config.java | 53 +- .../ConfigurationManagerImpl.java | 2 +- .../lb/LoadBalancingRulesManagerImpl.java | 36 +- .../network/router/CommandSetupHelper.java | 68 +-- .../VirtualNetworkApplianceManagerImpl.java | 522 ++++++++---------- .../root/health_checks/cpu_usage_check.py | 4 +- .../debian/root/health_checks/dhcp_check.py | 2 +- .../root/health_checks/disk_space_check.py | 2 +- systemvm/debian/root/monitorServices.py | 92 +-- .../views/infra/routers/RouterHealthCheck.vue | 3 +- 25 files changed, 543 insertions(+), 715 deletions(-) delete mode 100644 engine/schema/src/main/resources/META-INF/db/schema-42010to42100cleanup.sql diff --git a/api/src/main/java/com/cloud/configuration/ConfigurationService.java b/api/src/main/java/com/cloud/configuration/ConfigurationService.java index 32e31519ea7..438283136d2 100644 --- a/api/src/main/java/com/cloud/configuration/ConfigurationService.java +++ b/api/src/main/java/com/cloud/configuration/ConfigurationService.java @@ -108,36 +108,22 @@ public interface ConfigurationService { /** * Updates a service offering * - * @param serviceOfferingId - * @param userId - * @param name - * @param displayText - * @param offerHA - * @param useVirtualNetwork - * @param tags * @return updated service offering */ ServiceOffering updateServiceOffering(UpdateServiceOfferingCmd cmd); /** * Deletes a service offering - * - * @param userId - * @param serviceOfferingId */ boolean deleteServiceOffering(DeleteServiceOfferingCmd cmd); /** * Retrieve ID of domains for a service offering - * - * @param serviceOfferingId */ List getServiceOfferingDomains(Long serviceOfferingId); /** * Retrieve ID of domains for a service offering - * - * @param serviceOfferingId */ List getServiceOfferingZones(Long serviceOfferingId); @@ -147,7 +133,6 @@ public interface ConfigurationService { * @param cmd * - the command specifying diskOfferingId, name, description, tags * @return updated disk offering - * @throws */ DiskOffering updateDiskOffering(UpdateDiskOfferingCmd cmd); @@ -157,34 +142,22 @@ public interface ConfigurationService { * @param cmd * - the command specifying disk offering id * @return true or false - * @throws */ boolean deleteDiskOffering(DeleteDiskOfferingCmd cmd); /** * Creates a new disk offering - * - * @param domainId - * @param name - * @param description - * @param numGibibytes - * @param mirrored - * @param size * @return ID */ DiskOffering createDiskOffering(CreateDiskOfferingCmd cmd); /** * Retrieve ID of domains for a disk offering - * - * @param diskOfferingId */ List getDiskOfferingDomains(Long diskOfferingId); /** * Retrieve ID of domains for a disk offering - * - * @param diskOfferingId */ List getDiskOfferingZones(Long diskOfferingId); @@ -207,8 +180,6 @@ public interface ConfigurationService { * TODO * @param storageAccessGroups * @return the new pod if successful, null otherwise - * @throws - * @throws */ Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState, List storageAccessGroups); @@ -228,8 +199,7 @@ public interface ConfigurationService { /** * Updates a mutually exclusive IP range in the pod. * @param cmd - The command specifying pod ID, current Start IP, current End IP, new Start IP, new End IP. - * @throws com.cloud.exception.ConcurrentOperationException - * @return Success + * @throws com.cloud.exception.ConcurrentOperationException when this pod is already being accessed */ void updatePodIpRange(UpdatePodManagementNetworkIpRangeCmd cmd) throws ConcurrentOperationException; @@ -250,9 +220,6 @@ public interface ConfigurationService { /** * Edits a pod in the database. Will not allow you to edit pods that are being used anywhere in the system. - * - * @param UpdatePodCmd - * api command */ Pod editPod(UpdatePodCmd cmd); @@ -262,17 +229,12 @@ public interface ConfigurationService { * @param cmd * - the command containing podId * @return true or false - * @throws , */ boolean deletePod(DeletePodCmd cmd); /** * Creates a new zone - * - * @param cmd * @return the zone if successful, null otherwise - * @throws - * @throws */ DataCenter createZone(CreateZoneCmd cmd); @@ -295,22 +257,7 @@ public interface ConfigurationService { * Adds a VLAN to the database, along with an IP address range. Can add three types of VLANs: (1) zone-wide VLANs on * the * virtual public network (2) pod-wide direct attached VLANs (3) account-specific direct attached VLANs - * - * @param userId - * @param vlanType - * - either "DomR" (VLAN for a virtual public network) or "DirectAttached" (VLAN for IPs that will be - * directly - * attached to UserVMs) - * @param zoneId - * @param accountId - * @param podId - * @param add - * @param vlanId - * @param gateway - * @param startIP - * @param endIP * @throws ResourceAllocationException TODO - * @throws * @return The new Vlan object */ Vlan createVlanAndPublicIpRange(CreateVlanIpRangeCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException, @@ -325,9 +272,6 @@ public interface ConfigurationService { /** * Marks the account with the default zone-id. * - * @param accountName - * @param domainId - * @param defaultZoneId * @return The new account object */ Account markDefaultZone(String accountName, long domainId, long defaultZoneId); @@ -349,14 +293,12 @@ public interface ConfigurationService { /** * Retrieve ID of domains for a network offering * - * @param networkOfferingId */ List getNetworkOfferingDomains(Long networkOfferingId); /** * Retrieve ID of domains for a network offering * - * @param networkOfferingId */ List getNetworkOfferingZones(Long networkOfferingId); diff --git a/api/src/main/java/com/cloud/network/RouterHealthCheckResult.java b/api/src/main/java/com/cloud/network/RouterHealthCheckResult.java index eb65ae9088e..22a46ce9ecd 100644 --- a/api/src/main/java/com/cloud/network/RouterHealthCheckResult.java +++ b/api/src/main/java/com/cloud/network/RouterHealthCheckResult.java @@ -26,7 +26,7 @@ public interface RouterHealthCheckResult { String getCheckType(); - boolean getCheckResult(); + VirtualNetworkApplianceService.RouterHealthStatus getCheckResult(); Date getLastUpdateTime(); diff --git a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java index cb92739d283..a60f1d49336 100644 --- a/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java +++ b/api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java @@ -87,4 +87,8 @@ public interface VirtualNetworkApplianceService { Pair performRouterHealthChecks(long routerId); void collectNetworkStatistics(T router, Nic nic); + + enum RouterHealthStatus{ + SUCCESS, FAILED, WARNING, UNKNOWN; + } } diff --git a/api/src/main/java/com/cloud/network/lb/LoadBalancingRulesService.java b/api/src/main/java/com/cloud/network/lb/LoadBalancingRulesService.java index 3fc6028b977..0bf06be15d8 100644 --- a/api/src/main/java/com/cloud/network/lb/LoadBalancingRulesService.java +++ b/api/src/main/java/com/cloud/network/lb/LoadBalancingRulesService.java @@ -41,13 +41,23 @@ import com.cloud.utils.net.Ip; public interface LoadBalancingRulesService { /** * Create a load balancer rule from the given ipAddress/port to the given private port + * @param xId an existing UUID for this rule (for instance a device generated one) + * @param name + * @param description + * @param srcPortStart + * @param srcPortEnd + * @param defPortStart + * @param defPortEnd + * @param ipAddrId + * @param protocol + * @param algorithm + * @param networkId + * @param lbOwnerId * @param openFirewall - * TODO - * @param forDisplay TODO - * @param cmd - * the command specifying the ip address, public port, protocol, private port, and algorithm - * + * @param lbProtocol + * @param forDisplay * @return the newly created LoadBalancerVO if successful, null otherwise + * @throws NetworkRuleConflictException * @throws InsufficientAddressCapacityException */ LoadBalancer createPublicLoadBalancerRule(String xId, String name, String description, int srcPortStart, int srcPortEnd, int defPortStart, int defPortEnd, diff --git a/api/src/main/java/org/apache/cloudstack/api/response/RouterHealthCheckResultResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/RouterHealthCheckResultResponse.java index 00f1e4e3bb0..96364d0190c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/RouterHealthCheckResultResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/RouterHealthCheckResultResponse.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.api.response; import java.util.Date; +import com.cloud.network.VirtualNetworkApplianceService.RouterHealthStatus; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; @@ -35,9 +36,13 @@ public class RouterHealthCheckResultResponse extends BaseResponse { private String checkType; @SerializedName(ApiConstants.SUCCESS) - @Param(description = "result of the health check") + @Param(description = "result of the health check if available") private boolean result; + @SerializedName(ApiConstants.STATUS) + @Param(description = "the result of the health check in enum form: {SUCCESS, FAILURE, WARNING, UNKNOWN}") + private RouterHealthStatus state; + @SerializedName(ApiConstants.LAST_UPDATED) @Param(description = "the date this VPC was created") private Date lastUpdated; @@ -54,10 +59,14 @@ public class RouterHealthCheckResultResponse extends BaseResponse { return checkType; } - public boolean getResult() { + public Boolean getResult() { return result; } + public RouterHealthStatus getState() { + return state; + } + public Date getLastUpdated() { return lastUpdated; } @@ -74,10 +83,14 @@ public class RouterHealthCheckResultResponse extends BaseResponse { this.checkType = checkType; } - public void setResult(boolean result) { + public void setResult(Boolean result) { this.result = result; } + public void setState(RouterHealthStatus state) { + this.state = state; + } + public void setLastUpdated(Date lastUpdated) { this.lastUpdated = lastUpdated; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index adce5f2f8b4..31b08429cc4 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -76,44 +76,52 @@ public interface NetworkOrchestrationService { */ Long RVRHandoverTime = 10000L; - ConfigKey MinVRVersion = new ConfigKey(String.class, MinVRVersionCK, "Advanced", "4.10.0", + ConfigKey MinVRVersion = new ConfigKey<>(String.class, MinVRVersionCK, "Advanced", "4.10.0", "What version should the Virtual Routers report", true, ConfigKey.Scope.Zone, null); - ConfigKey NetworkLockTimeout = new ConfigKey(Integer.class, NetworkLockTimeoutCK, "Network", "600", - "Lock wait timeout (seconds) while implementing network", true, Scope.Global, null); + ConfigKey NetworkLockTimeout = new ConfigKey<>(Integer.class, NetworkLockTimeoutCK, "Network", "600", + "Lock wait timeout (seconds) while implementing network", true, Scope.Global, null); - ConfigKey DeniedRoutes = new ConfigKey(String.class, "denied.routes", "Network", "", + ConfigKey DeniedRoutes = new ConfigKey<>(String.class, "denied.routes", "Network", "", "Routes that are denied, can not be used for Static Routes creation for the VPC Private Gateway", true, ConfigKey.Scope.Zone, null); - ConfigKey GuestDomainSuffix = new ConfigKey(String.class, GuestDomainSuffixCK, "Network", "cloud.internal", - "Default domain name for vms inside virtualized networks fronted by router", true, ConfigKey.Scope.Zone, null); + ConfigKey GuestDomainSuffix = new ConfigKey<>(String.class, GuestDomainSuffixCK, "Network", "cloud.internal", + "Default domain name for vms inside virtualized networks fronted by router", true, ConfigKey.Scope.Zone, null); - ConfigKey NetworkThrottlingRate = new ConfigKey("Network", Integer.class, NetworkThrottlingRateCK, "200", - "Default data transfer rate in megabits per second allowed in network.", true, ConfigKey.Scope.Zone); + ConfigKey NetworkThrottlingRate = new ConfigKey<>("Network", Integer.class, NetworkThrottlingRateCK, "200", + "Default data transfer rate in megabits per second allowed in network.", true, ConfigKey.Scope.Zone); - ConfigKey PromiscuousMode = new ConfigKey("Advanced", Boolean.class, "network.promiscuous.mode", "false", + ConfigKey PromiscuousMode = new ConfigKey<>("Advanced", Boolean.class, "network.promiscuous.mode", "false", "Whether to allow or deny promiscuous mode on nics for applicable network elements such as for vswitch/dvswitch portgroups.", true); - ConfigKey MacAddressChanges = new ConfigKey("Advanced", Boolean.class, "network.mac.address.changes", "true", + ConfigKey MacAddressChanges = new ConfigKey<>("Advanced", Boolean.class, "network.mac.address.changes", "true", "Whether to allow or deny mac address changes on nics for applicable network elements such as for vswitch/dvswitch porgroups.", true); - ConfigKey ForgedTransmits = new ConfigKey("Advanced", Boolean.class, "network.forged.transmits", "true", + ConfigKey ForgedTransmits = new ConfigKey<>("Advanced", Boolean.class, "network.forged.transmits", "true", "Whether to allow or deny forged transmits on nics for applicable network elements such as for vswitch/dvswitch portgroups.", true); - ConfigKey MacLearning = new ConfigKey("Advanced", Boolean.class, "network.mac.learning", "false", + ConfigKey MacLearning = new ConfigKey<>("Advanced", Boolean.class, "network.mac.learning", "false", "Whether to allow or deny MAC learning on nics for applicable network elements such as for dvswitch portgroups.", true); - ConfigKey RollingRestartEnabled = new ConfigKey("Advanced", Boolean.class, "network.rolling.restart", "true", + ConfigKey RollingRestartEnabled = new ConfigKey<>("Advanced", Boolean.class, "network.rolling.restart", "true", "Whether to allow or deny rolling restart of network routers.", true); - static final ConfigKey TUNGSTEN_ENABLED = new ConfigKey<>(Boolean.class, "tungsten.plugin.enable", "Advanced", "false", + ConfigKey TUNGSTEN_ENABLED = new ConfigKey<>(Boolean.class, "tungsten.plugin.enable", "Advanced", "false", "Indicates whether to enable the Tungsten plugin", false, ConfigKey.Scope.Zone, null); - static final ConfigKey NSX_ENABLED = new ConfigKey<>(Boolean.class, "nsx.plugin.enable", "Advanced", "false", + ConfigKey NSX_ENABLED = new ConfigKey<>(Boolean.class, "nsx.plugin.enable", "Advanced", "false", "Indicates whether to enable the NSX plugin", false, ConfigKey.Scope.Zone, null); ConfigKey NETRIS_ENABLED = new ConfigKey<>(Boolean.class, "netris.plugin.enable", "Advanced", "false", "Indicates whether to enable the Netris plugin", false, ConfigKey.Scope.Zone, null); + ConfigKey NETWORK_LB_HAPROXY_MAX_CONN = new ConfigKey<>( + "Network", + Integer.class, + "network.loadbalancer.haproxy.max.conn", + "4096", + "Load Balancer(haproxy) maximum number of concurrent connections(global max)", + true, + Scope.Global); List setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault) throws ConcurrentOperationException; @@ -129,7 +137,7 @@ public interface NetworkOrchestrationService { * configures the provided dhcp options on the given nic. * @param network of the nic * @param nicId - * @param extraDhcpOptions + * @param extraDhcpOptions a map of rank:value pairs */ void configureExtraDhcpOptions(Network network, long nicId, Map extraDhcpOptions); @@ -158,16 +166,15 @@ public interface NetworkOrchestrationService { Pair implementNetwork(long networkId, DeployDestination dest, ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; - Map getExtraDhcpOptions(long nicId); - /** * Returns all extra dhcp options which are set on the provided nic * @param nicId * @return map which maps the dhcp value on it's option code */ + Map getExtraDhcpOptions(long nicId); + /** * prepares vm nic change for migration - * * This method will be called in migration transaction before the vm migration. * @param vm * @param dest @@ -176,7 +183,6 @@ public interface NetworkOrchestrationService { /** * commit vm nic change for migration - * * This method will be called in migration transaction after the successful * vm migration. * @param src @@ -186,7 +192,6 @@ public interface NetworkOrchestrationService { /** * rollback vm nic change for migration - * * This method will be called in migaration transaction after vm migration * failure. * @param src @@ -266,7 +271,7 @@ public interface NetworkOrchestrationService { void releaseNic(VirtualMachineProfile vmProfile, Nic nic) throws ConcurrentOperationException, ResourceUnavailableException; NicProfile createNicForVm(Network network, NicProfile requested, ReservationContext context, VirtualMachineProfile vmProfile, boolean prepare) - throws InsufficientVirtualNetworkCapacityException, InsufficientAddressCapacityException, ConcurrentOperationException, InsufficientCapacityException, + throws ConcurrentOperationException, InsufficientCapacityException, ResourceUnavailableException; NetworkProfile convertNetworkToNetworkProfile(long networkId); @@ -277,7 +282,7 @@ public interface NetworkOrchestrationService { boolean shutdownNetworkElementsAndResources(ReservationContext context, boolean b, Network network); void implementNetworkElementsAndResources(DeployDestination dest, ReservationContext context, Network network, NetworkOffering findById) - throws ConcurrentOperationException, InsufficientAddressCapacityException, ResourceUnavailableException, InsufficientCapacityException; + throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException; Map finalizeServicesAndProvidersForNetwork(NetworkOffering offering, Long physicalNetworkId); diff --git a/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java b/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java index eebf764289d..5909d098db8 100644 --- a/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java +++ b/engine/components-api/src/main/java/com/cloud/configuration/ConfigurationManager.java @@ -56,10 +56,10 @@ import com.cloud.utils.net.NetUtils; */ public interface ConfigurationManager { - public static final String MESSAGE_CREATE_POD_IP_RANGE_EVENT = "Message.CreatePodIpRange.Event"; - public static final String MESSAGE_DELETE_POD_IP_RANGE_EVENT = "Message.DeletePodIpRange.Event"; - public static final String MESSAGE_CREATE_VLAN_IP_RANGE_EVENT = "Message.CreateVlanIpRange.Event"; - public static final String MESSAGE_DELETE_VLAN_IP_RANGE_EVENT = "Message.DeleteVlanIpRange.Event"; + String MESSAGE_CREATE_POD_IP_RANGE_EVENT = "Message.CreatePodIpRange.Event"; + String MESSAGE_DELETE_POD_IP_RANGE_EVENT = "Message.DeletePodIpRange.Event"; + String MESSAGE_CREATE_VLAN_IP_RANGE_EVENT = "Message.CreateVlanIpRange.Event"; + String MESSAGE_DELETE_VLAN_IP_RANGE_EVENT = "Message.DeleteVlanIpRange.Event"; public static final ConfigKey AllowNonRFC1918CompliantIPs = new ConfigKey<>(Boolean.class, "allow.non.rfc1918.compliant.ips", "Advanced", "false", @@ -72,8 +72,9 @@ public interface ConfigurationManager { true, ConfigKey.Scope.Global); /** - * @param offering - * @return + * Is this for a VPC + * @param offering the offering to check + * @return true or false */ boolean isOfferingForVpc(NetworkOffering offering); @@ -83,80 +84,12 @@ public interface ConfigurationManager { /** * Updates a configuration entry with a new value - * @param userId - * @param name - * @param category - * @param value - * @param scope - * @param id */ String updateConfiguration(long userId, String name, String category, String value, ConfigKey.Scope scope, Long id); -// /** -// * Creates a new service offering -// * -// * @param name -// * @param cpu -// * @param ramSize -// * @param speed -// * @param displayText -// * @param localStorageRequired -// * @param offerHA -// * @param domainId -// * @param volatileVm -// * @param hostTag -// * @param networkRate -// * @param id -// * @param useVirtualNetwork -// * @param deploymentPlanner -// * @param details -// * @param bytesReadRate -// * @param bytesWriteRate -// * @param iopsReadRate -// * @param iopsWriteRate -// * @return ID -// */ -// ServiceOfferingVO createServiceOffering(long userId, boolean isSystem, VirtualMachine.Type vm_typeType, String name, int cpu, int ramSize, int speed, String displayText, boolean localStorageRequired, -// boolean offerHA, boolean limitResourceUse, boolean volatileVm, String tags, Long domainId, String hostTag, Integer networkRate, String deploymentPlanner, Map details, -// Long bytesReadRate, Long bytesWriteRate, Long iopsReadRate, Long iopsWriteRate); - -// /** -// * Creates a new disk offering -// * -// * @param domainId -// * @param name -// * @param description -// * @param numGibibytes -// * @param tags -// * @param isCustomized -// * @param localStorageRequired -// * @param isDisplayOfferingEnabled -// * @param isCustomizedIops (is admin allowing users to set custom iops?) -// * @param minIops -// * @param maxIops -// * @param bytesReadRate -// * @param bytesWriteRate -// * @param iopsReadRate -// * @param iopsWriteRate -// * @return newly created disk offering -// */ -// DiskOfferingVO createDiskOffering(Long domainId, String name, String description, Long numGibibytes, String tags, boolean isCustomized, -// boolean localStorageRequired, boolean isDisplayOfferingEnabled, Boolean isCustomizedIops, Long minIops, Long maxIops, -// Long bytesReadRate, Long bytesWriteRate, Long iopsReadRate, Long iopsWriteRate); - /** * Creates a new pod - * - * @param userId - * @param podName - * @param zone - * @param gateway - * @param cidr - * @param startIp - * @param endIp - * @param allocationState * @param skipGatewayOverlapCheck (true if it is ok to not validate that gateway IP address overlap with Start/End IP of the POD) - * @param storageAccessGroups * @return Pod */ HostPodVO createPod(long userId, String podName, DataCenter zone, String gateway, String cidr, String startIp, String endIp, String allocationState, @@ -164,7 +97,6 @@ public interface ConfigurationManager { /** * Creates a new zone - * * @param userId * @param zoneName * @param dns1 @@ -172,16 +104,18 @@ public interface ConfigurationManager { * @param internalDns1 * @param internalDns2 * @param guestCidr + * @param domain + * @param domainId * @param zoneType * @param allocationState * @param networkDomain * @param isSecurityGroupEnabled + * @param isLocalStorageEnabled * @param ip6Dns1 * @param ip6Dns2 + * @param isEdge * @param storageAccessGroups * @return - * @throws - * @throws */ DataCenterVO createZone(long userId, String zoneName, String dns1, String dns2, String internalDns1, String internalDns2, String guestCidr, String domain, Long domainId, NetworkType zoneType, String allocationState, String networkDomain, boolean isSecurityGroupEnabled, boolean isLocalStorageEnabled, String ip6Dns1, @@ -191,9 +125,7 @@ public interface ConfigurationManager { * Deletes a VLAN from the database, along with all of its IP addresses. Will not delete VLANs that have allocated * IP addresses. * - * @param userId - * @param vlanDbId - * @param caller + * @param caller TODO * @return success/failure */ VlanVO deleteVlanAndPublicIpRange(long userId, long vlanDbId, Account caller); @@ -204,31 +136,39 @@ public interface ConfigurationManager { /** * Creates a new network offering - * * @param name * @param displayText * @param trafficType * @param tags * @param specifyVlan + * @param availability * @param networkRate * @param serviceProviderMap * @param isDefault * @param type * @param systemOnly * @param serviceOfferingId - * @param conserveMode ; + * @param conserveMode + * @param serviceCapabilityMap * @param specifyIpRanges - * @param isPersistent ; + * @param isPersistent * @param details + * @param egressDefaultPolicy + * @param maxconn + * @param enableKeepAlive * @param forVpc * @param forTungsten * @param forNsx * @param forNetris + * @param networkMode * @param domainIds * @param zoneIds - * @return network offering object + * @param enableOffering + * @param internetProtocol + * @param routingMode + * @param specifyAsNumber + * @return the network offering */ - NetworkOfferingVO createNetworkOffering(String name, String displayText, TrafficType trafficType, String tags, boolean specifyVlan, Availability availability, Integer networkRate, Map> serviceProviderMap, boolean isDefault, Network.GuestType type, boolean systemOnly, Long serviceOfferingId, boolean conserveMode, Map> serviceCapabilityMap, boolean specifyIpRanges, boolean isPersistent, @@ -245,7 +185,6 @@ public interface ConfigurationManager { /** * Release dedicated virtual ip ranges of a domain. * - * @param domainId * @return success/failure */ boolean releaseDomainSpecificVirtualRanges(Domain domain); @@ -253,7 +192,6 @@ public interface ConfigurationManager { /** * Release dedicated virtual ip ranges of an account. * - * @param accountId * @return success/failure */ boolean releaseAccountSpecificVirtualRanges(Account account); @@ -261,16 +199,7 @@ public interface ConfigurationManager { /** * Edits a pod in the database. Will not allow you to edit pods that are being used anywhere in the system. * - * @param id - * @param name - * @param startIp - * @param endIp - * @param gateway - * @param netmask - * @param allocationState * @return Pod - * @throws - * @throws */ Pod editPod(long id, String name, String startIp, String endIp, String gateway, String netmask, String allocationState); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 146643bae22..38cf6ed05be 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -454,7 +454,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra SearchBuilder AssignIpAddressSearch; SearchBuilder AssignIpAddressFromPodVlanSearch; - HashMap _lastNetworkIdsToFree = new HashMap(); + HashMap _lastNetworkIdsToFree = new HashMap<>(); private void updateRouterDefaultDns(final VirtualMachineProfile vmProfile, final NicProfile nicProfile) { if (!Type.DomainRouter.equals(vmProfile.getType()) || !nicProfile.isDefaultNic()) { @@ -492,8 +492,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @DB public boolean configure(final String name, final Map params) throws ConfigurationException { // populate providers - final Map> defaultSharedNetworkOfferingProviders = new HashMap>(); - final Set defaultProviders = new HashSet(); + final Map> defaultSharedNetworkOfferingProviders = new HashMap<>(); + final Set defaultProviders = new HashSet<>(); final Set tungstenProvider = new HashSet<>(); defaultProviders.add(Network.Provider.VirtualRouter); @@ -512,11 +512,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra defaultIsolatedNetworkOfferingProviders.put(Service.PortForwarding, defaultProviders); defaultIsolatedNetworkOfferingProviders.put(Service.Vpn, defaultProviders); - final Map> defaultSharedSGEnabledNetworkOfferingProviders = new HashMap>(); + final Map> defaultSharedSGEnabledNetworkOfferingProviders = new HashMap<>(); defaultSharedSGEnabledNetworkOfferingProviders.put(Service.Dhcp, defaultProviders); defaultSharedSGEnabledNetworkOfferingProviders.put(Service.Dns, defaultProviders); defaultSharedSGEnabledNetworkOfferingProviders.put(Service.UserData, defaultProviders); - final Set sgProviders = new HashSet(); + final Set sgProviders = new HashSet<>(); sgProviders.add(Provider.SecurityGroupProvider); defaultSharedSGEnabledNetworkOfferingProviders.put(Service.SecurityGroup, sgProviders); @@ -529,7 +529,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra defaultTungstenSharedSGEnabledNetworkOfferingProviders.put(Service.SecurityGroup, tungstenProvider); - final Map> defaultIsolatedSourceNatEnabledNetworkOfferingProviders = new HashMap>(); + final Map> defaultIsolatedSourceNatEnabledNetworkOfferingProviders = new HashMap<>(); defaultProviders.clear(); defaultProviders.add(Network.Provider.VirtualRouter); defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Dhcp, defaultProviders); @@ -543,7 +543,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.PortForwarding, defaultProviders); defaultIsolatedSourceNatEnabledNetworkOfferingProviders.put(Service.Vpn, defaultProviders); - final Map> defaultVPCOffProviders = new HashMap>(); + final Map> defaultVPCOffProviders = new HashMap<>(); defaultProviders.clear(); defaultProviders.add(Network.Provider.VPCVirtualRouter); defaultVPCOffProviders.put(Service.Dhcp, defaultProviders); @@ -560,11 +560,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { - NetworkOfferingVO offering = null; + NetworkOfferingVO offering; //#1 - quick cloud network offering if (_networkOfferingDao.findByUniqueName(NetworkOffering.QuickCloudNoServices) == null) { offering = _configMgr.createNetworkOffering(NetworkOffering.QuickCloudNoServices, "Offering for QuickCloud with no services", TrafficType.Guest, null, true, - Availability.Optional, null, new HashMap>(), true, Network.GuestType.Shared, false, null, true, null, true, + Availability.Optional, null, new HashMap<>(), true, Network.GuestType.Shared, false, null, true, null, true, false, null, false, null, true, false, false, false, false, null, null, null, true, null, null, false); } @@ -622,11 +622,11 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } //#8 - network offering with internal lb service - final Map> internalLbOffProviders = new HashMap>(); - final Set defaultVpcProvider = new HashSet(); + final Map> internalLbOffProviders = new HashMap<>(); + final Set defaultVpcProvider = new HashSet<>(); defaultVpcProvider.add(Network.Provider.VPCVirtualRouter); - final Set defaultInternalLbProvider = new HashSet(); + final Set defaultInternalLbProvider = new HashSet<>(); defaultInternalLbProvider.add(Network.Provider.InternalLbVm); internalLbOffProviders.put(Service.Dhcp, defaultVpcProvider); @@ -646,12 +646,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra _networkOfferingDao.update(offering.getId(), offering); } - final Map> netscalerServiceProviders = new HashMap>(); - final Set vrProvider = new HashSet(); + final Map> netscalerServiceProviders = new HashMap<>(); + final Set vrProvider = new HashSet<>(); vrProvider.add(Provider.VirtualRouter); - final Set sgProvider = new HashSet(); + final Set sgProvider = new HashSet<>(); sgProvider.add(Provider.SecurityGroupProvider); - final Set nsProvider = new HashSet(); + final Set nsProvider = new HashSet<>(); nsProvider.add(Provider.Netscaler); netscalerServiceProviders.put(Service.Dhcp, vrProvider); netscalerServiceProviders.put(Service.Dns, vrProvider); @@ -660,10 +660,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra netscalerServiceProviders.put(Service.StaticNat, nsProvider); netscalerServiceProviders.put(Service.Lb, nsProvider); - final Map> serviceCapabilityMap = new HashMap>(); - final Map elb = new HashMap(); + final Map> serviceCapabilityMap = new HashMap<>(); + final Map elb = new HashMap<>(); elb.put(Capability.ElasticLb, "true"); - final Map eip = new HashMap(); + final Map eip = new HashMap<>(); eip.put(Capability.ElasticIp, "true"); serviceCapabilityMap.put(Service.Lb, elb); serviceCapabilityMap.put(Service.StaticNat, eip); @@ -726,11 +726,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return true; } - @Override - public boolean stop() { - return true; - } - protected NetworkOrchestrator() { setStateMachine(); } @@ -772,7 +767,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } - final List networks = new ArrayList(); + final List networks = new ArrayList<>(); long related = -1; @@ -895,7 +890,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra Arrays.fill(deviceIds, false); List> profilesList = getOrderedNetworkNicProfileMapping(networks); - final List nics = new ArrayList(size); + final List nics = new ArrayList<>(size); NicProfile defaultNic = null; Network nextNetwork = null; for (Pair networkNicPair : profilesList) { @@ -972,16 +967,16 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra for (final Map.Entry> network : networks.entrySet()) { List requestedProfiles = network.getValue(); if (requestedProfiles == null) { - requestedProfiles = new ArrayList(); + requestedProfiles = new ArrayList<>(); } if (requestedProfiles.isEmpty()) { requestedProfiles.add(null); } for (final NicProfile requested : requestedProfiles) { - profilesList.add(new Pair(network.getKey(), requested)); + profilesList.add(new Pair<>(network.getKey(), requested)); } } - profilesList.sort(new Comparator>() { + profilesList.sort(new Comparator<>() { @Override public int compare(Pair pair1, Pair pair2) { int profile1Order = Integer.MAX_VALUE; @@ -1064,7 +1059,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } private NicVO persistNicAfterRaceCheck(final NicVO nic, final Long networkId, final NicProfile profile, int deviceId) { - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public NicVO doInTransaction(TransactionStatus status) { NicVO vo = _nicDao.findNonPlaceHolderByIp4AddressAndNetworkId(profile.getIPv4Address(), networkId); @@ -1085,7 +1080,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra logger.debug("Allocating nic for vm {} in network {} with requested profile {}", vm.getVirtualMachine(), network, requested); final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, ntwkVO.getGuruName()); - NicVO vo = null; + NicVO vo; boolean retryIpAllocation; do { retryIpAllocation = false; @@ -1161,7 +1156,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra _nicDao.update(vo.getId(), vo); setMtuInVRNicProfile(networks, network.getTrafficType(), vmNic); } - return new Pair(vmNic, Integer.valueOf(deviceId)); + return new Pair<>(vmNic, Integer.valueOf(deviceId)); } private boolean isNicAllocatedForProviderPublicNetworkOnVR(Network network, NicProfile requested, VirtualMachineProfile vm, Provider provider) { @@ -1443,7 +1438,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra Pair implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context, final boolean isRouter) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { - Pair implemented = null; + Pair implemented; if (!isRouter) { implemented = implementNetwork(networkId, dest, context); } else { @@ -1454,7 +1449,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra // in issues. In order to avoid it, implementNetwork() call for VR is replaced with below code. final NetworkVO network = _networksDao.findById(networkId); final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); - implemented = new Pair(guru, network); + implemented = new Pair<>(guru, network); } return implemented; } @@ -1542,7 +1537,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @DB public Pair implementNetwork(final long networkId, final DeployDestination dest, final ReservationContext context) throws ConcurrentOperationException, ResourceUnavailableException, InsufficientCapacityException { - final Pair implemented = new Pair(null, null); + final Pair implemented = new Pair<>(null, null); NetworkVO network = _networksDao.findById(networkId); final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); @@ -1609,10 +1604,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return implemented; } catch (final NoTransitionException e) { logger.error(e.getMessage()); - return new Pair(null, null); + return new Pair<>(null, null); } catch (final CloudRuntimeException | OperationTimedoutException e) { logger.error("Caught exception: {}", e.getMessage()); - return new Pair(null, null); + return new Pair<>(null, null); } finally { if (implemented.first() == null) { logger.debug("Cleaning up because we're unable to implement the network {}", network); @@ -1656,7 +1651,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra && (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.SourceNat) || _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Gateway)) && (network.getGuestType() == Network.GuestType.Isolated || network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced)) { - List ips = null; + List ips; final Account owner = _entityMgr.findById(Account.class, network.getAccountId()); if (network.getVpcId() != null) { ips = _ipAddressDao.listByAssociatedVpc(network.getVpcId(), true); @@ -2145,7 +2140,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra // we have to implement default nics first - to ensure that default network elements start up first in multiple //nics case // (need for setting DNS on Dhcp to domR's Ip4 address) - Collections.sort(nics, new Comparator() { + Collections.sort(nics, new Comparator<>() { @Override public int compare(final NicVO nic1, final NicVO nic2) { @@ -2183,7 +2178,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); final NicVO nic = _nicDao.findById(nicId); - NicProfile profile = null; + NicProfile profile; if (nic.getReservationStrategy() == Nic.ReservationStrategy.Start) { nic.setState(Nic.State.Reserving); nic.setReservationId(context.getReservationId()); @@ -2344,7 +2339,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra vm.addNic(profile); } - final List addedURIs = new ArrayList(); + final List addedURIs = new ArrayList<>(); if (guestNetworkId != null) { final List publicIps = _ipAddressDao.listByAssociatedNetwork(guestNetworkId, null); for (final IPAddressVO userIp : publicIps) { @@ -2992,7 +2987,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final String networkDomainFinal = networkDomain; final String vlanIdFinal = vlanId; final Boolean subdomainAccessFinal = subdomainAccess; - final Network network = Transaction.execute(new TransactionCallback() { + final Network network = Transaction.execute(new TransactionCallback<>() { @Override public Network doInTransaction(final TransactionStatus status) { Long physicalNetworkId = null; @@ -3097,7 +3092,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra userNetwork.setNetworkCidrSize(networkCidrSize); final List networks = setupNetwork(owner, ntwkOff, userNetwork, plan, name, displayText, true, domainId, aclType, subdomainAccessFinal, vpcId, isDisplayNetworkEnabled); - Network network = null; + Network network; if (networks == null || networks.isEmpty()) { throw new CloudRuntimeException("Fail to create a network"); } else { @@ -3226,10 +3221,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final boolean success = shutdownNetworkElementsAndResources(context, cleanupElements, network); final NetworkVO networkFinal = network; - final boolean result = Transaction.execute(new TransactionCallback() { + final boolean result = Transaction.execute(new TransactionCallback<>() { @Override public Boolean doInTransaction(final TransactionStatus status) { - boolean result = false; + boolean result; if (success) { logger.debug("Network {} is shutdown successfully, cleaning up corresponding resources now.", networkFinal); @@ -3447,7 +3442,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkVO networkFinal = network; try { - final List deletedVlanRangeToPublish = Transaction.execute(new TransactionCallback>() { + final List deletedVlanRangeToPublish = Transaction.execute(new TransactionCallback<>() { @Override public List doInTransaction(TransactionStatus status) { final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, networkFinal.getGuruName()); @@ -3503,7 +3498,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra publishDeletedVlanRanges(deletedVlanRangeToPublish); if (_networksDao.findById(network.getId()) == null) { // remove its related ACL permission - final Pair, Long> networkMsg = new Pair, Long>(Network.class, networkFinal.getId()); + final Pair, Long> networkMsg = new Pair<>(Network.class, networkFinal.getId()); _messageBus.publish(_name, EntityManager.MESSAGE_REMOVE_ENTITY_EVENT, PublishScope.LOCAL, networkMsg); } UsageEventUtils.publishNetworkDeletion(network); @@ -3586,9 +3581,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra public void reallyRun() { try { - final List shutdownList = new ArrayList(); + final List shutdownList = new ArrayList<>(); final long currentTime = System.currentTimeMillis() / 1000; - final HashMap stillFree = new HashMap(); + final HashMap stillFree = new HashMap<>(); final List networkIds = _networksDao.findNetworksToGarbageCollect(); final int netGcWait = NumbersUtil.parseInt(_configDao.getValue(NetworkGcWait.key()), 60); @@ -3958,7 +3953,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public List listVmNics(final long vmId, final Long nicId, final Long networkId, String keyword) { - List result = null; + List result; if (keyword == null || keyword.isEmpty()) { if (nicId == null && networkId == null) { @@ -4001,8 +3996,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra if (dc.getNetworkType() == NetworkType.Basic) { final List nics = _nicDao.listByVmId(vmInstance.getId()); final NetworkVO network = _networksDao.findById(nics.get(0).getNetworkId()); - final LinkedHashMap> profiles = new LinkedHashMap>(); - profiles.put(network, new ArrayList()); + final LinkedHashMap> profiles = new LinkedHashMap<>(); + profiles.put(network, new ArrayList<>()); Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { @Override @@ -4136,7 +4131,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra // Mark all static rules as revoked and apply them on the backend (not in the DB) final List firewallStaticNatRules = _firewallDao.listByNetworkAndPurpose(network.getId(), Purpose.StaticNat); - final List staticNatRules = new ArrayList(); + final List staticNatRules = new ArrayList<>(); logger.debug("Releasing {} static nat rules for network {} as a part of shutdownNetworkRules", firewallStaticNatRules.size(), network); for (final FirewallRuleVO firewallStaticNatRule : firewallStaticNatRules) { @@ -4258,7 +4253,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra // Get all ip addresses, mark as releasing and release them on the backend final List userIps = _ipAddressDao.listByAssociatedNetwork(network.getId(), null); - final List publicIpsToRelease = new ArrayList(); + final List publicIpsToRelease = new ArrayList<>(); if (userIps != null && !userIps.isEmpty()) { for (final IPAddressVO userIp : userIps) { userIp.setState(IpAddress.State.Releasing); @@ -4307,7 +4302,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final String dataCenter = startup.getDataCenter(); - long dcId = -1; + long dcId; DataCenterVO dc = _dcDao.findByName(dataCenter); if (dc == null) { try { @@ -4324,7 +4319,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra logger.debug("Host's hypervisorType is: {}", hypervisorType); - final List networkInfoList = new ArrayList(); + final List networkInfoList = new ArrayList<>(); // list all physicalnetworks in the zone & for each get the network names final List physicalNtwkList = _physicalNetworkDao.listByZone(dcId); @@ -4403,8 +4398,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public Map finalizeServicesAndProvidersForNetwork(final NetworkOffering offering, final Long physicalNetworkId) { - final Map svcProviders = new HashMap(); - final Map> providerSvcs = new HashMap>(); + final Map svcProviders = new HashMap<>(); + final Map> providerSvcs = new HashMap<>(); final List servicesMap = _ntwkOfferingSrvcDao.listByNetworkOfferingId(offering.getId()); final boolean checkPhysicalNetwork = physicalNetworkId != null ? true : false; @@ -4434,7 +4429,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra svcProviders.put(service, provider); List l = providerSvcs.get(provider); if (l == null) { - providerSvcs.put(provider, l = new ArrayList()); + providerSvcs.put(provider, l = new ArrayList<>()); } l.add(service); } @@ -4444,7 +4439,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra private List getNetworkProviders(final long networkId) { final List providerNames = _ntwkSrvcDao.getDistinctProviders(networkId); - final List providers = new ArrayList(); + final List providers = new ArrayList<>(); for (final String providerName : providerNames) { providers.add(Network.Provider.getProvider(providerName)); } @@ -4539,7 +4534,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra @Override public List getNicProfiles(final Long vmId, HypervisorType hypervisorType) { final List nics = _nicDao.listByVmId(vmId); - final List profiles = new ArrayList(); + final List profiles = new ArrayList<>(); if (nics != null) { for (final Nic nic : nics) { @@ -4605,12 +4600,12 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } private Map> getServiceProvidersMap(final long networkId) { - final Map> map = new HashMap>(); + final Map> map = new HashMap<>(); final List nsms = _ntwkSrvcDao.getServicesInNetwork(networkId); for (final NetworkServiceMapVO nsm : nsms) { Set providers = map.get(Service.getService(nsm.getService())); if (providers == null) { - providers = new HashSet(); + providers = new HashSet<>(); } providers.add(Provider.getProvider(nsm.getProvider())); map.put(Service.getService(nsm.getService()), providers); @@ -4622,14 +4617,14 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra public List getProvidersForServiceInNetwork(final Network network, final Service service) { final Map> service2ProviderMap = getServiceProvidersMap(network.getId()); if (service2ProviderMap.get(service) != null) { - final List providers = new ArrayList(service2ProviderMap.get(service)); + final List providers = new ArrayList<>(service2ProviderMap.get(service)); return providers; } return null; } protected List getElementForServiceInNetwork(final Network network, final Service service) { - final List elements = new ArrayList(); + final List elements = new ArrayList<>(); final List providers = getProvidersForServiceInNetwork(network, service); //Only support one provider now if (providers == null) { @@ -4663,7 +4658,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final List lbElements = getElementForServiceInNetwork(network, Service.Lb); NetworkElement lbElement = null; if (lbElements.size() > 1) { - String providerName = null; + String providerName; //get network offering details final NetworkOffering off = _entityMgr.findById(NetworkOffering.class, network.getNetworkOfferingId()); if (lbScheme == Scheme.Public) { @@ -4751,7 +4746,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } final String finalSelectedIp = selectedIp; - final NicVO vo = Transaction.execute(new TransactionCallback() { + final NicVO vo = Transaction.execute(new TransactionCallback<>() { @Override public NicVO doInTransaction(TransactionStatus status) { if (StringUtils.isBlank(macAddress)) { @@ -4901,9 +4896,9 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return NetworkOrchestrationService.class.getSimpleName(); } - public static final ConfigKey NetworkGcWait = new ConfigKey(Integer.class, "network.gc.wait", "Advanced", "600", + public static final ConfigKey NetworkGcWait = new ConfigKey<>(Integer.class, "network.gc.wait", "Advanced", "600", "Time (in seconds) to wait before shutting down a network that's not in used", false, Scope.Global, null); - public static final ConfigKey NetworkGcInterval = new ConfigKey(Integer.class, "network.gc.interval", "Advanced", "600", + public static final ConfigKey NetworkGcInterval = new ConfigKey<>(Integer.class, "network.gc.interval", "Advanced", "600", "Seconds to wait before checking for networks to shutdown", true, Scope.Global, null); @Override @@ -4911,6 +4906,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return new ConfigKey[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes, GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled, - TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED }; + TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN}; } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultVO.java b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultVO.java index 9803ccb6a4b..204ef2d1538 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultVO.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/RouterHealthCheckResultVO.java @@ -29,6 +29,7 @@ import javax.persistence.Temporal; import javax.persistence.TemporalType; import com.cloud.network.RouterHealthCheckResult; +import com.cloud.network.VirtualNetworkApplianceService; import com.cloud.utils.StringUtils; @Entity @@ -49,7 +50,7 @@ public class RouterHealthCheckResultVO implements RouterHealthCheckResult { private String checkType; @Column(name = "check_result") - private boolean checkResult; + private VirtualNetworkApplianceService.RouterHealthStatus checkResult; @Temporal(TemporalType.TIMESTAMP) @Column(name = "last_update", updatable = true, nullable = true) @@ -87,7 +88,7 @@ public class RouterHealthCheckResultVO implements RouterHealthCheckResult { } @Override - public boolean getCheckResult() { + public VirtualNetworkApplianceService.RouterHealthStatus getCheckResult() { return checkResult; } @@ -105,7 +106,7 @@ public class RouterHealthCheckResultVO implements RouterHealthCheckResult { return checkDetails; } - public void setCheckResult(boolean checkResult) { + public void setCheckResult(VirtualNetworkApplianceService.RouterHealthStatus checkResult) { this.checkResult = checkResult; } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100cleanup.sql deleted file mode 100644 index 9165565f544..00000000000 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100cleanup.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. - ---; --- Schema upgrade from 4.20.1.0 to 4.21.0.0 ---; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index cf3fe2ed772..0538e45d1dc 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -19,5 +19,9 @@ -- Schema upgrade from 4.21.0.0 to 4.22.0.0 --; + +-- health check status as enum +CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', 'check_result', 'varchar(16) NOT NULL COMMENT "check executions result: SUCCESS, FAILURE, WARNING, UNKNOWN"'); + -- Increase length of scripts_version column to 128 due to md5sum to sha512sum change CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)'); diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index 26151ab5b58..88eca1d28de 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -378,23 +378,21 @@ public class ConfigKey { if (type.isAssignableFrom(Boolean.class)) { return (T)Boolean.valueOf(value); } else if (type.isAssignableFrom(Integer.class)) { - return (T)new Integer(Integer.parseInt(value) * multiplier.intValue()); + return (T)Integer.valueOf(Integer.parseInt(value) * multiplier.intValue()); } else if (type.isAssignableFrom(Long.class)) { - return (T)new Long(Long.parseLong(value) * multiplier.longValue()); + return (T)Long.valueOf(Long.parseLong(value) * multiplier.longValue()); } else if (type.isAssignableFrom(Short.class)) { - return (T)new Short(Short.parseShort(value)); + return (T)Short.valueOf(Short.parseShort(value)); } else if (type.isAssignableFrom(String.class)) { return (T)value; } else if (type.isAssignableFrom(Float.class)) { - return (T)new Float(Float.parseFloat(value) * multiplier.floatValue()); + return (T)Float.valueOf(Float.parseFloat(value) * multiplier.floatValue()); } else if (type.isAssignableFrom(Double.class)) { - return (T)new Double(Double.parseDouble(value) * multiplier.doubleValue()); - } else if (type.isAssignableFrom(String.class)) { - return (T)value; + return (T)Double.valueOf(Double.parseDouble(value) * multiplier.doubleValue()); } else if (type.isAssignableFrom(Date.class)) { return (T)Date.valueOf(value); } else if (type.isAssignableFrom(Character.class)) { - return (T)new Character(value.charAt(0)); + return (T)Character.valueOf(value.charAt(0)); } else { throw new CloudRuntimeException("Unsupported data type for config values: " + type); } diff --git a/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java b/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java index c02d8cf67aa..f895ba2944c 100644 --- a/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java +++ b/plugins/network-elements/elastic-loadbalancer/src/main/java/com/cloud/network/lb/ElasticLoadBalancerManagerImpl.java @@ -32,6 +32,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd; import org.apache.cloudstack.config.ApiServiceConfiguration; +import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.springframework.stereotype.Component; @@ -201,7 +202,7 @@ public class ElasticLoadBalancerManagerImpl extends ManagerBase implements Elast NetworkOffering offering = _networkOfferingDao.findById(guestNetworkId); String maxconn = null; if (offering.getConcurrentConnections() == null) { - maxconn = _configDao.getValue(Config.NetworkLBHaproxyMaxConn.key()); + maxconn = NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value().toString(); } else { maxconn = offering.getConcurrentConnections().toString(); } diff --git a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java index 4512b908556..d979a4b3033 100644 --- a/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java +++ b/plugins/network-elements/internal-loadbalancer/src/main/java/org/apache/cloudstack/network/lb/InternalLoadBalancerVMManagerImpl.java @@ -488,7 +488,7 @@ public class InternalLoadBalancerVMManagerImpl extends ManagerBase implements In final NetworkOffering offering = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()); String maxconn = null; if (offering.getConcurrentConnections() == null) { - maxconn = _configDao.getValue(Config.NetworkLBHaproxyMaxConn.key()); + maxconn = NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value().toString(); } else { maxconn = offering.getConcurrentConnections().toString(); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 64d6e8b6929..bf65bde0203 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -5100,7 +5100,17 @@ public class ApiResponseHelper implements ResponseGenerator { healthCheckResponse.setObjectName("routerhealthchecks"); healthCheckResponse.setCheckName(hcResult.getCheckName()); healthCheckResponse.setCheckType(hcResult.getCheckType()); - healthCheckResponse.setResult(hcResult.getCheckResult()); + switch (hcResult.getCheckResult()) { + case SUCCESS: + healthCheckResponse.setResult(true); + break; + case FAILED: + healthCheckResponse.setResult(false); + break; + default: + // no result if not definite + } + healthCheckResponse.setState(hcResult.getCheckResult()); healthCheckResponse.setLastUpdated(hcResult.getLastUpdateTime()); healthCheckResponse.setDetails(hcResult.getParsedCheckDetails()); responses.add(healthCheckResponse); diff --git a/server/src/main/java/com/cloud/configuration/Config.java b/server/src/main/java/com/cloud/configuration/Config.java index 6a05ce227a7..d728c251af7 100644 --- a/server/src/main/java/com/cloud/configuration/Config.java +++ b/server/src/main/java/com/cloud/configuration/Config.java @@ -255,14 +255,6 @@ public enum Config { "8081", "Load Balancer(haproxy) stats port number.", null), - NetworkLBHaproxyMaxConn( - "Network", - ManagementServer.class, - Integer.class, - "network.loadbalancer.haproxy.max.conn", - "4096", - "Load Balancer(haproxy) maximum number of concurrent connections(global max)", - null), NetworkRouterRpFilter( "Network", ManagementServer.class, @@ -1714,11 +1706,11 @@ public enum Config { private static final HashMap> s_scopeLevelConfigsMap = new HashMap<>(); static { - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Zone.getBitValue(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Cluster.getBitValue(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.StoragePool.getBitValue(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Account.getBitValue(), new ArrayList()); - s_scopeLevelConfigsMap.put(ConfigKey.Scope.Global.getBitValue(), new ArrayList()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Zone.getBitValue(), new ArrayList<>()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Cluster.getBitValue(), new ArrayList<>()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.StoragePool.getBitValue(), new ArrayList<>()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Account.getBitValue(), new ArrayList<>()); + s_scopeLevelConfigsMap.put(ConfigKey.Scope.Global.getBitValue(), new ArrayList<>()); for (Config c : Config.values()) { //Creating group of parameters per each level (zone/cluster/pool/account) @@ -1731,23 +1723,22 @@ public enum Config { } } - private static final HashMap> Configs = new HashMap>(); + private static final HashMap> Configs = new HashMap<>(); static { // Add categories - Configs.put("Alert", new ArrayList()); - Configs.put("Storage", new ArrayList()); - Configs.put("Snapshots", new ArrayList()); - Configs.put("Network", new ArrayList()); - Configs.put("Usage", new ArrayList()); - Configs.put("Console Proxy", new ArrayList()); - Configs.put("Advanced", new ArrayList()); - Configs.put("Usage", new ArrayList()); - Configs.put("Developer", new ArrayList()); - Configs.put("Hidden", new ArrayList()); - Configs.put("Account Defaults", new ArrayList()); - Configs.put("Domain Defaults", new ArrayList()); - Configs.put("Project Defaults", new ArrayList()); - Configs.put("Secure", new ArrayList()); + Configs.put("Account Defaults", new ArrayList<>()); + Configs.put("Advanced", new ArrayList<>()); + Configs.put("Alert", new ArrayList<>()); + Configs.put("Console Proxy", new ArrayList<>()); + Configs.put("Developer", new ArrayList<>()); + Configs.put("Domain Defaults", new ArrayList<>()); + Configs.put("Hidden", new ArrayList<>()); + Configs.put("Network", new ArrayList<>()); + Configs.put("Secure", new ArrayList<>()); + Configs.put("Snapshots", new ArrayList<>()); + Configs.put("Storage", new ArrayList<>()); + Configs.put("Usage", new ArrayList<>()); + Configs.put("Project Defaults", new ArrayList<>()); // Add values into HashMap for (Config c : Config.values()) { @@ -1758,11 +1749,11 @@ public enum Config { } } - private Config(String category, Class componentClass, Class type, String name, String defaultValue, String description, String range) { + Config(String category, Class componentClass, Class type, String name, String defaultValue, String description, String range) { this(category, componentClass, type, name, defaultValue, description, range, null, null); } - private Config(String category, Class componentClass, Class type, String name, String defaultValue, String description, String range, ConfigKey.Kind kind, String options) { + Config(String category, Class componentClass, Class type, String name, String defaultValue, String description, String range, ConfigKey.Kind kind, String options) { _category = category; _componentClass = componentClass; _type = type; @@ -1867,7 +1858,7 @@ public enum Config { public static List getCategories() { Object[] keys = Configs.keySet().toArray(); - List categories = new ArrayList(); + List categories = new ArrayList<>(); for (Object key : keys) { categories.add((String)key); } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 409dd056ef7..2c5a931a831 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -6810,7 +6810,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (lbServiceCapabilityMap != null && !lbServiceCapabilityMap.isEmpty()) { maxconn = cmd.getMaxconnections(); if (maxconn == null) { - maxconn = Integer.parseInt(_configDao.getValue(Config.NetworkLBHaproxyMaxConn.key())); + maxconn = NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value(); } } if (cmd.getKeepAliveEnabled() != null && cmd.getKeepAliveEnabled()) { diff --git a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java index d8ea9b55071..5ceebf06dd8 100644 --- a/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java +++ b/server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java @@ -2138,32 +2138,33 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements //Included revoked rules to remove the rules of ips which are in revoke state List rules = _firewallDao.listByIpAndPurpose(ipId, Purpose.LoadBalancing); + if (deleteRulesFails(caller, callerUserId, rules)) return false; + return true; + } + + private boolean deleteRulesFails(Account caller, long callerUserId, List rules) { if (rules != null) { - logger.debug("Found " + rules.size() + " lb rules to cleanup"); + logger.debug("Found {} lb rules to cleanup", rules.size()); for (FirewallRule rule : rules) { - boolean result = deleteLoadBalancerRule(rule.getId(), true, caller, callerUserId, false); - if (result == false) { - logger.warn("Unable to remove load balancer rule {}", rule); - return false; - } + if (deleteRuleFails(caller, callerUserId, rule)) return true; } } - return true; + return false; + } + + private boolean deleteRuleFails(Account caller, long callerUserId, FirewallRule rule) { + boolean result = deleteLoadBalancerRule(rule.getId(), true, caller, callerUserId, false); + if (result == false) { + logger.warn("Unable to remove load balancer rule {}", rule); + return true; + } + return false; } @Override public boolean removeAllLoadBalanacersForNetwork(long networkId, Account caller, long callerUserId) { List rules = _firewallDao.listByNetworkAndPurposeAndNotRevoked(networkId, Purpose.LoadBalancing); - if (rules != null) { - logger.debug("Found " + rules.size() + " lb rules to cleanup"); - for (FirewallRule rule : rules) { - boolean result = deleteLoadBalancerRule(rule.getId(), true, caller, callerUserId, false); - if (result == false) { - logger.warn("Unable to remove load balancer rule {}", rule); - return false; - } - } - } + if (deleteRulesFails(caller, callerUserId, rules)) return false; return true; } @@ -2792,5 +2793,4 @@ public class LoadBalancingRulesManagerImpl extends ManagerBase implements } return null; } - } diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index 278c2531411..2ca6ca0d0be 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -28,19 +28,19 @@ import java.util.Set; import javax.inject.Inject; -import com.cloud.agent.api.HandleCksIsoCommand; -import com.cloud.network.rules.PortForwardingRuleVO; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.network.BgpPeer; import org.apache.cloudstack.network.BgpPeerTO; import org.apache.cloudstack.network.dao.BgpPeerDetailsDao; + import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import com.cloud.agent.api.HandleCksIsoCommand; import com.cloud.agent.api.SetupGuestNetworkCommand; import com.cloud.agent.api.routing.CreateIpAliasCommand; import com.cloud.agent.api.routing.DeleteIpAliasCommand; @@ -122,6 +122,7 @@ import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRule.Purpose; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.PortForwardingRule; +import com.cloud.network.rules.PortForwardingRuleVO; import com.cloud.network.rules.StaticNat; import com.cloud.network.rules.StaticNatRule; import com.cloud.network.vpc.NetworkACLItem; @@ -255,8 +256,8 @@ public class CommandSetupHelper { } public void createApplyVpnUsersCommand(final List users, final VirtualRouter router, final Commands cmds) { - final List addUsers = new ArrayList(); - final List removeUsers = new ArrayList(); + final List addUsers = new ArrayList<>(); + final List removeUsers = new ArrayList<>(); for (final VpnUser user : users) { if (user.getState() == VpnUser.State.Add || user.getState() == VpnUser.State.Active) { addUsers.add(user); @@ -319,7 +320,7 @@ public class CommandSetupHelper { public void configDnsMasq(final VirtualRouter router, final Network network, final Commands cmds) { final DataCenterVO dcVo = _dcDao.findById(router.getDataCenterId()); final List ipAliasVOList = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.State.active); - final List ipList = new ArrayList(); + final List ipList = new ArrayList<>(); final NicVO router_guest_nic = _nicDao.findByNtwkIdAndInstanceId(network.getId(), router.getId()); final String cidr = NetUtils.getCidrFromGatewayAndNetmask(router_guest_nic.getIPv4Gateway(), router_guest_nic.getIPv4Netmask()); @@ -384,9 +385,9 @@ public class CommandSetupHelper { final NicProfile nicProfile = new NicProfile(nic, guestNetwork, nic.getBroadcastUri(), nic.getIsolationUri(), _networkModel.getNetworkRate(guestNetwork.getId(), router.getId()), _networkModel.isSecurityGroupSupportedInNetwork(guestNetwork), _networkModel.getNetworkTag(router.getHypervisorType(), guestNetwork)); final NetworkOffering offering = _networkOfferingDao.findById(guestNetwork.getNetworkOfferingId()); - String maxconn = null; + String maxconn; if (offering.getConcurrentConnections() == null) { - maxconn = _configDao.getValue(Config.NetworkLBHaproxyMaxConn.key()); + maxconn = NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value().toString(); } else { maxconn = offering.getConcurrentConnections().toString(); } @@ -408,7 +409,7 @@ public class CommandSetupHelper { } public void createApplyPortForwardingRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { - final List rulesTO = new ArrayList(); + final List rulesTO = new ArrayList<>(); if (rules != null) { for (final PortForwardingRule rule : rules) { _rulesDao.loadSourceCidrs((PortForwardingRuleVO) rule); @@ -418,7 +419,7 @@ public class CommandSetupHelper { } } - SetPortForwardingRulesCommand cmd = null; + SetPortForwardingRulesCommand cmd; if (router.getVpcId() != null) { cmd = new SetPortForwardingRulesVpcCommand(rulesTO); @@ -436,7 +437,7 @@ public class CommandSetupHelper { } public void createApplyStaticNatRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { - final List rulesTO = new ArrayList(); + final List rulesTO = new ArrayList<>(); if (rules != null) { for (final StaticNatRule rule : rules) { final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId()); @@ -455,11 +456,11 @@ public class CommandSetupHelper { } public void createApplyFirewallRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { - final List rulesTO = new ArrayList(); + final List rulesTO = new ArrayList<>(); String systemRule = null; Boolean defaultEgressPolicy = false; if (rules != null) { - if (rules.size() > 0) { + if (!rules.isEmpty()) { if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System) { systemRule = String.valueOf(FirewallRule.FirewallRuleType.System); } @@ -506,9 +507,9 @@ public class CommandSetupHelper { String systemRule = null; final NetworkVO network = _networkDao.findById(guestNetworkId); final NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); - Boolean defaultEgressPolicy = offering.isEgressDefaultPolicy();; + Boolean defaultEgressPolicy = offering.isEgressDefaultPolicy(); if (rules != null) { - if (rules.size() > 0) { + if (!rules.isEmpty()) { if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System) { systemRule = String.valueOf(FirewallRule.FirewallRuleType.System); } @@ -543,14 +544,13 @@ public class CommandSetupHelper { } public void createFirewallRulesCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { - final List rulesTO = new ArrayList(); + final List rulesTO = new ArrayList<>(); String systemRule = null; Boolean defaultEgressPolicy = false; if (rules != null) { - if (rules.size() > 0) { - if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System) { - systemRule = String.valueOf(FirewallRule.FirewallRuleType.System); - } + boolean isSystemFirewallEgressRule = !rules.isEmpty() && rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System; + if (isSystemFirewallEgressRule) { + systemRule = String.valueOf(FirewallRule.FirewallRuleType.System); } for (final FirewallRule rule : rules) { _rulesDao.loadSourceCidrs((FirewallRuleVO) rule); @@ -597,7 +597,7 @@ public class CommandSetupHelper { final NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); Boolean defaultEgressPolicy = offering.isEgressDefaultPolicy(); if (rules != null) { - if (rules.size() > 0) { + if (!rules.isEmpty()) { if (rules.get(0).getTrafficType() == FirewallRule.TrafficType.Egress && rules.get(0).getType() == FirewallRule.FirewallRuleType.System) { systemRule = String.valueOf(FirewallRule.FirewallRuleType.System); } @@ -638,7 +638,7 @@ public class CommandSetupHelper { public void createNetworkACLsCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId, final boolean privateGateway) { - final List rulesTO = new ArrayList(); + final List rulesTO = new ArrayList<>(); String guestVlan = null; final Network guestNtwk = _networkDao.findById(guestNetworkId); final URI uri = guestNtwk.getBroadcastUri(); @@ -687,7 +687,7 @@ public class CommandSetupHelper { } public void createApplyStaticNatCommands(final List rules, final VirtualRouter router, final Commands cmds, final long guestNetworkId) { - final List rulesTO = new ArrayList(); + final List rulesTO = new ArrayList<>(); if (rules != null) { for (final StaticNat rule : rules) { final IpAddress sourceIp = _networkModel.getIp(rule.getSourceIpAddressId()); @@ -811,7 +811,7 @@ public class CommandSetupHelper { Boolean addSourceNat = null; // Ensure that in multiple vlans case we first send all ip addresses of // vlan1, then all ip addresses of vlan2, etc.. - final Map> vlanIpMap = new HashMap>(); + final Map> vlanIpMap = new HashMap<>(); for (final PublicIpAddress ipAddress : ips) { String vlanTag = ipAddress.getVlanTag(); if (Objects.isNull(vlanTag)) { @@ -819,7 +819,7 @@ public class CommandSetupHelper { } ArrayList ipList = vlanIpMap.get(vlanTag); if (ipList == null) { - ipList = new ArrayList(); + ipList = new ArrayList<>(); } // VR doesn't support release for sourceNat IP address; so reset the // state @@ -847,7 +847,7 @@ public class CommandSetupHelper { final List ipAddrList = vlanAndIp.getValue(); // Source nat ip address should always be sent first - Collections.sort(ipAddrList, new Comparator() { + Collections.sort(ipAddrList, new Comparator<>() { @Override public int compare(final PublicIpAddress o1, final PublicIpAddress o2) { final boolean s1 = o1.isSourceNat(); @@ -896,7 +896,7 @@ public class CommandSetupHelper { } ipsToSend[i++] = ip; if (ipAddr.isSourceNat()) { - sourceNatIpAdd = new Pair(ip, ipAddr.getNetworkId()); + sourceNatIpAdd = new Pair<>(ip, ipAddr.getNetworkId()); addSourceNat = add; } @@ -934,12 +934,12 @@ public class CommandSetupHelper { // Ensure that in multiple vlans case we first send all ip addresses of // vlan1, then all ip addresses of vlan2, etc.. - final Map> vlanIpMap = new HashMap>(); + final Map> vlanIpMap = new HashMap<>(); for (final PublicIpAddress ipAddress : ips) { final String vlanTag = ipAddress.getVlanTag(); ArrayList ipList = vlanIpMap.get(vlanTag); if (ipList == null) { - ipList = new ArrayList(); + ipList = new ArrayList<>(); } // domR doesn't support release for sourceNat IP address; so reset // the state @@ -952,7 +952,7 @@ public class CommandSetupHelper { final List nics = _nicDao.listByVmId(router.getId()); String baseMac = null; - Map vlanMacAddress = new HashMap();; + Map vlanMacAddress = new HashMap<>(); Long guestNetworkId = null; for (final NicVO nic : nics) { final NetworkVO nw = _networkDao.findById(nic.getNetworkId()); @@ -973,7 +973,7 @@ public class CommandSetupHelper { final String vlanTagKey = vlanAndIp.getKey(); final List ipAddrList = vlanAndIp.getValue(); // Source nat ip address should always be sent first - Collections.sort(ipAddrList, new Comparator() { + Collections.sort(ipAddrList, new Comparator<>() { @Override public int compare(final PublicIpAddress o1, final PublicIpAddress o2) { final boolean s1 = o1.isSourceNat(); @@ -1001,7 +1001,7 @@ public class CommandSetupHelper { final String vlanId = ipAddr.getVlanTag(); final String vlanGateway = ipAddr.getGateway(); final String vlanNetmask = ipAddr.getNetmask(); - String vifMacAddress = null; + String vifMacAddress; final String vlanTag = BroadcastDomainType.getValue(BroadcastDomainType.fromString(ipAddr.getVlanTag())); if (vlanMacAddress.containsKey(vlanTag)) { vifMacAddress = vlanMacAddress.get(vlanTag); @@ -1077,7 +1077,7 @@ public class CommandSetupHelper { private Map getVlanLastIpMap(Long vpcId, Long guestNetworkId) { // for network if the ips does not have any rules, then only last ip - final Map vlanLastIpMap = new HashMap(); + final Map vlanLastIpMap = new HashMap<>(); final List userIps; if (vpcId != null) { userIps = _ipAddressDao.listByAssociatedVpc(vpcId, null); @@ -1145,12 +1145,12 @@ public class CommandSetupHelper { // Ensure that in multiple vlans case we first send all ip addresses of // vlan1, then all ip addresses of vlan2, etc.. - final Map> vlanIpMap = new HashMap>(); + final Map> vlanIpMap = new HashMap<>(); for (final PrivateIpAddress ipAddress : ips) { final String vlanTag = ipAddress.getBroadcastUri(); ArrayList ipList = vlanIpMap.get(vlanTag); if (ipList == null) { - ipList = new ArrayList(); + ipList = new ArrayList<>(); } ipList.add(ipAddress); diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index 19cec194345..eb5995b56f8 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -27,7 +27,6 @@ import java.security.NoSuchAlgorithmException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Calendar; import java.util.Collections; import java.util.Date; @@ -48,6 +47,9 @@ import java.util.regex.Pattern; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; + import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.alert.AlertService.AlertType; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -72,6 +74,7 @@ import org.apache.cloudstack.network.topology.NetworkTopologyContext; import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.usage.UsageUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -264,8 +267,6 @@ import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.NicIpAliasDao; import com.cloud.vm.dao.NicIpAliasVO; import com.cloud.vm.dao.VMInstanceDetailsDao; -import com.google.gson.JsonSyntaxException; -import com.google.gson.reflect.TypeToken; /** * VirtualNetworkApplianceManagerImpl manages the different types of virtual @@ -282,60 +283,60 @@ Configurable, StateListener _guestOSNeedGatewayOnNonDefaultNetwork = new HashSet<>(); private boolean _disableRpFilter = false; private int _routerExtraPublicNics = 2; @@ -417,13 +413,11 @@ Configurable, StateListener configs = _configDao.getConfiguration("AgentManager", params); - _routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); - _routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); + int routerRamSize = NumbersUtil.parseInt(configs.get("router.ram.size"), DEFAULT_ROUTER_VM_RAMSIZE); + int routerCpuMHz = NumbersUtil.parseInt(configs.get("router.cpu.mhz"), DEFAULT_ROUTER_CPU_MHZ); _routerExtraPublicNics = NumbersUtil.parseInt(_configDao.getValue(Config.RouterExtraPublicNics.key()), 2); - final String guestOSString = configs.get("network.dhcp.nondefaultnetwork.setgateway.guestos"); - if (guestOSString != null) { - final String[] guestOSList = guestOSString.split(","); - for (final String os : guestOSList) { - _guestOSNeedGatewayOnNonDefaultNetwork.add(os); - } - } - String value = configs.get("router.stats.interval"); _routerStatsInterval = NumbersUtil.parseInt(value, 300); @@ -573,7 +559,7 @@ Configurable, StateListener(_rvrStatusUpdatePoolSize * 1000); + _vrUpdateQueue = new LinkedBlockingQueue<>(_rvrStatusUpdatePoolSize * 1000); _rvrStatusUpdateExecutor = Executors.newFixedThreadPool(_rvrStatusUpdatePoolSize, new NamedThreadFactory("RedundantRouterStatusMonitor")); @@ -591,12 +577,12 @@ Configurable, StateListener 0) { @@ -727,7 +713,7 @@ Configurable, StateListener routers = _routerDao.listByStateAndNetworkType(VirtualMachine.State.Running, GuestType.Isolated, mgmtSrvrId); - logger.debug("Found " + routers.size() + " running routers. "); + logger.debug("Found {} running routers. ", routers.size()); for (final DomainRouterVO router : routers) { collectNetworkStatistics(router, null); @@ -813,7 +799,7 @@ Configurable, StateListener ipList = new ArrayList(); + final List ipList = new ArrayList<>(); for (final Site2SiteVpnConnectionVO conn : conns) { if (conn.getState() != Site2SiteVpnConnection.State.Connected && conn.getState() != Site2SiteVpnConnection.State.Disconnected && conn.getState() != Site2SiteVpnConnection.State.Connecting) { @@ -824,26 +810,24 @@ Configurable, StateListener routers) { - final Set checkedNetwork = new HashSet(); + final Set checkedNetwork = new HashSet<>(); for (final DomainRouterVO router : routers) { if (!router.getIsRedundantRouter()) { continue; @@ -1005,18 +964,12 @@ Configurable, StateListener routers) { - final Map networkRouterMaps = new HashMap(); + final Map networkRouterMaps = new HashMap<>(); for (final DomainRouterVO router : routers) { final List routerGuestNtwkIds = _routerDao.getRouterNetworks(router.getId()); final Long vpcId = router.getVpcId(); - if (vpcId != null || routerGuestNtwkIds.size() > 0) { + if (vpcId != null || !routerGuestNtwkIds.isEmpty()) { Long routerGuestNtwkId = vpcId != null ? vpcId : routerGuestNtwkIds.get(0); if (router.getRedundantState() == RedundantState.PRIMARY) { if (networkRouterMaps.containsKey(routerGuestNtwkId)) { @@ -1088,12 +1041,7 @@ Configurable, StateListener networks = new ArrayList<>(); for (Vpc vpc : _vpcDao.listAll()) { List vpcNetworks = _networkDao.listByVpc(vpc.getId()); - if (vpcNetworks.size() > 0) { + if (!vpcNetworks.isEmpty()) { networks.add(vpcNetworks.get(0)); } } @@ -1176,7 +1124,6 @@ Configurable, StateListener getFailingChecks(DomainRouterVO router, GetRouterMonitorResultsAnswer answer) { if (answer == null) { - logger.warn("Unable to fetch monitor results for router " + router); - resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Communication failed"); - return Arrays.asList(CONNECTIVITY_TEST); + logger.warn("Unable to fetch monitor results for router {}", router); + resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.UNKNOWN, RouterHealthStatus.UNKNOWN, "Communication failed"); + return List.of(CONNECTIVITY_TEST); } else if (!answer.getResult()) { logger.warn("Failed to fetch monitor results from router " + router + " with details: " + answer.getDetails()); if (StringUtils.isNotBlank(answer.getDetails()) && answer.getDetails().equalsIgnoreCase(READONLY_FILESYSTEM_ERROR)) { - resetRouterHealthChecksAndConnectivity(router.getId(), true, false, "Failed to write: " + answer.getDetails()); - return Arrays.asList(FILESYSTEM_WRITABLE_TEST); + resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.SUCCESS, RouterHealthStatus.FAILED, "Failed to write: " + answer.getDetails()); + return List.of(FILESYSTEM_WRITABLE_TEST); } else { - resetRouterHealthChecksAndConnectivity(router.getId(), false, false, "Failed to fetch results with details: " + answer.getDetails()); - return Arrays.asList(CONNECTIVITY_TEST); + resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.FAILED, RouterHealthStatus.UNKNOWN, "Failed to fetch results with details: " + answer.getDetails()); + return List.of(CONNECTIVITY_TEST); } } else { - resetRouterHealthChecksAndConnectivity(router.getId(), true, true, "Successfully fetched data"); + resetRouterHealthChecksAndConnectivity(router.getId(), RouterHealthStatus.SUCCESS, RouterHealthStatus.SUCCESS, "Successfully fetched data"); updateDbHealthChecksFromRouterResponse(router, answer.getMonitoringResults()); return answer.getFailingChecks(); } } private void handleFailingChecks(DomainRouterVO router, List failingChecks) { - if (failingChecks == null || failingChecks.size() == 0) { + if (CollectionUtils.isEmpty(failingChecks)) { return; } @@ -1256,17 +1203,16 @@ Configurable, StateListener> getHealthChecksFromDb(long routerId) { @@ -1336,13 +1281,13 @@ Configurable, StateListener checkData, - final Map> checksInDb) { - boolean success = Boolean.parseBoolean(checkData.get("success")); + final String checkName, final String checkType, final Map checkData, + final Map> checksInDb) { + RouterHealthStatus success = getRouterHealthStatus(checkData.get("success")); Date lastUpdate = new Date(Long.parseLong(checkData.get("lastUpdate"))); double lastRunDuration = Double.parseDouble(checkData.get("lastRunDuration")); String message = checkData.get("message"); @@ -1394,6 +1339,16 @@ Configurable, StateListener updateDbHealthChecksFromRouterResponse(final DomainRouterVO router, final String monitoringResult) { + private void updateDbHealthChecksFromRouterResponse(final DomainRouterVO router, final String monitoringResult) { if (StringUtils.isBlank(monitoringResult)) { logger.warn("Attempted parsing empty monitoring results string for router {}", router); - return Collections.emptyList(); + return; } try { logger.debug("Parsing and updating DB health check data for router: {} with data: {}", router, monitoringResult); final Type t = new TypeToken>>>() {}.getType(); final Map>> checks = GsonHelper.getGson().fromJson(monitoringResult, t); - return parseHealthCheckResults(checks, router); + parseHealthCheckResults(checks, router); } catch (JsonSyntaxException ex) { logger.error("Unable to parse the result of health checks due to " + ex.getLocalizedMessage(), ex); } - return Collections.emptyList(); } private GetRouterMonitorResultsAnswer fetchAndUpdateRouterHealthChecks(DomainRouterVO router, boolean performFreshChecks) { @@ -1470,14 +1424,15 @@ Configurable, StateListener failingChecks = getFailingChecks(router, answer); handleFailingChecks(router, failingChecks); - return new Pair(success, resultDetails); + return new Pair<>(success, resultDetails); } protected class UpdateRouterHealthChecksConfigTask extends ManagedContextRunnable { @@ -1584,7 +1539,7 @@ Configurable, StateListener routers = _routerDao.listByStateAndManagementServer(VirtualMachine.State.Running, mgmtSrvrId); - logger.debug("Found " + routers.size() + " running routers. "); + logger.debug("Found {} running routers. ", routers.size()); for (final DomainRouterVO router : routers) { GetRouterMonitorResultsAnswer answer = performBasicTestsOnRouter(router); @@ -1643,7 +1598,7 @@ Configurable, StateListener stickinessPolicyVOs = lbStickinessPolicyDao.listByLoadBalancerId(loadBalancingRuleId, false); - if (stickinessPolicyVOs != null && stickinessPolicyVOs.size() > 0) { + if (stickinessPolicyVOs != null && !stickinessPolicyVOs.isEmpty()) { StringBuilder stickiness = new StringBuilder(); for (LBStickinessPolicyVO stickinessVO : stickinessPolicyVOs) { stickiness.append(stickinessVO.getMethodName()).append(" "); @@ -1724,13 +1676,13 @@ Configurable, StateListener loadBalancerVOs = this.getLBRules(routerJoinVO); for (FirewallRuleVO firewallRuleVO : loadBalancerVOs) { List vmMapVOs = _loadBalancerVMMapDao.listByLoadBalancerId(firewallRuleVO.getId(), false); - if (vmMapVOs.size() > 0) { + if (!vmMapVOs.isEmpty()) { final NetworkOffering offering = _networkOfferingDao.findById(_networkDao.findById(routerJoinVO.getNetworkId()).getNetworkOfferingId()); if (offering.getConcurrentConnections() == null) { - loadBalancingData.append("maxconn=").append(_configDao.getValue(Config.NetworkLBHaproxyMaxConn.key())); + loadBalancingData.append("maxconn=").append(NetworkOrchestrationService.NETWORK_LB_HAPROXY_MAX_CONN.value()); } else { - loadBalancingData.append("maxconn=").append(offering.getConcurrentConnections().toString()); + loadBalancingData.append("maxconn=").append(offering.getConcurrentConnections()); } loadBalancingData.append(",sourcePortStart=").append(firewallRuleVO.getSourcePortStart()) @@ -1782,7 +1734,7 @@ Configurable, StateListener sbvm = userVmJoinDao.createSearchBuilder(); sbvm.and("networkId", sbvm.entity().getNetworkId(), SearchCriteria.Op.EQ); @@ -1866,22 +1818,11 @@ Configurable, StateListener details = _vmDetailsDao.listDetailsKeyPairs(router.getId()); router.setDetails(details); - // 2) Prepare boot loader elements related with Control network + // 2) Prepare bootloader elements related with Control network final StringBuilder buf = profile.getBootArgsBuilder(); buf.append(" template=domP"); buf.append(" name=").append(profile.getHostName()); - if (Boolean.valueOf(_configDao.getValue("system.vm.random.password"))) { + if (Boolean.parseBoolean(_configDao.getValue("system.vm.random.password"))) { buf.append(" vmpassword=").append(_configDao.getValue("system.vm.password")); } String msPublicKey = _configDao.getValue("ssh.publickey"); @@ -2009,14 +1961,14 @@ Configurable, StateListener acntq = QueryBuilder.create(UserVO.class); acntq.and(acntq.entity().getUsername(), SearchCriteria.Op.EQ, "baremetal-system-account"); final UserVO user = acntq.find(); if (user == null) { - logger.warn(String - .format("global setting[baremetal.provision.done.notification] is enabled but user baremetal-system-account is not found. Baremetal provision done notification will not be enabled")); + logger.warn("global setting[baremetal.provision.done.notification] is enabled but user baremetal-system-account is not found. Baremetal provision done notification will not be enabled"); } else { buf.append(String.format(" baremetalnotificationsecuritykey=%s", user.getSecretKey())); buf.append(String.format(" baremetalnotificationapikey=%s", user.getApiKey())); @@ -2150,7 +2097,7 @@ Configurable, StateListener services = new ArrayList(); + final List services = new ArrayList<>(); if (_networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dhcp, provider) || _networkModel.isProviderSupportServiceInNetwork(network.getId(), Service.Dns, provider)) { final MonitoringServiceVO dhcpService = _monitorServiceDao.getServiceByName(MonitoringService.Service.Dhcp.toString()); @@ -2418,7 +2357,7 @@ Configurable, StateListener servicesTO = new ArrayList(); + final List servicesTO = new ArrayList<>(); for (final MonitoringServiceVO service : services) { final MonitorServiceTO serviceTO = new MonitorServiceTO(service.getService(), service.getProcessName(), service.getServiceName(), service.getServicePath(), service.getServicePidFile(), service.isDefaultService()); @@ -2439,7 +2378,7 @@ Configurable, StateListener publicIps = getPublicIpsToApply(provider, guestNetworkId); - final List firewallRulesEgress = new ArrayList(); + final List firewallRulesEgress = new ArrayList<>(); final List ipv6firewallRules = new ArrayList<>(); // Fetch firewall Egress rules. @@ -2559,11 +2496,11 @@ Configurable, StateListener vpns = new ArrayList(); - final List pfRules = new ArrayList(); - final List staticNatFirewallRules = new ArrayList(); - final List staticNats = new ArrayList(); - final List firewallRulesIngress = new ArrayList(); + final List vpns = new ArrayList<>(); + final List pfRules = new ArrayList<>(); + final List staticNatFirewallRules = new ArrayList<>(); + final List staticNats = new ArrayList<>(); + final List firewallRulesIngress = new ArrayList<>(); // Get information about all the rules (StaticNats and // StaticNatRules; PFVPN to reapply on domR start) @@ -2622,7 +2559,7 @@ Configurable, StateListener staticNatRules = new ArrayList(); + final List staticNatRules = new ArrayList<>(); for (final FirewallRule rule : staticNatFirewallRules) { staticNatRules.add(_rulesMgr.buildStaticNatRule(rule, false)); } @@ -2644,18 +2581,18 @@ Configurable, StateListener dhcpCapabilities = _networkSvc.getNetworkOfferingServiceCapabilities( _networkOfferingDao.findById(_networkDao.findById(guestNetworkId).getNetworkOfferingId()), Service.Dhcp); final String supportsMultipleSubnets = dhcpCapabilities.get(Network.Capability.DhcpAccrossMultipleSubnets); - if (supportsMultipleSubnets != null && Boolean.valueOf(supportsMultipleSubnets)) { + if (Boolean.parseBoolean(supportsMultipleSubnets)) { final List revokedIpAliasVOs = _nicIpAliasDao.listByNetworkIdAndState(guestNetworkId, NicIpAlias.State.revoked); - logger.debug("Found" + revokedIpAliasVOs.size() + "ip Aliases to revoke on the router as a part of dhcp configuration"); + logger.debug("Found {} ip Aliases to revoke on the router as a part of dhcp configuration", revokedIpAliasVOs.size()); removeRevokedIpAliasFromDb(revokedIpAliasVOs); final List aliasVOs = _nicIpAliasDao.listByNetworkIdAndState(guestNetworkId, NicIpAlias.State.active); - logger.debug("Found" + aliasVOs.size() + "ip Aliases to apply on the router as a part of dhcp configuration"); - final List activeIpAliasTOs = new ArrayList(); + logger.debug("Found {} ip Aliases to apply on the router as a part of dhcp configuration", aliasVOs.size()); + final List activeIpAliasTOs = new ArrayList<>(); for (final NicIpAliasVO aliasVO : aliasVOs) { activeIpAliasTOs.add(new IpAliasTO(aliasVO.getIp4Address(), aliasVO.getNetmask(), aliasVO.getAliasCount().toString())); } - if (activeIpAliasTOs.size() != 0) { + if (!activeIpAliasTOs.isEmpty()) { _commandSetupHelper.createIpAlias(router, activeIpAliasTOs, guestNetworkId, cmds); _commandSetupHelper.configDnsMasq(router, _networkDao.findById(guestNetworkId), cmds); } @@ -2668,7 +2605,7 @@ Configurable, StateListener lbs = _loadBalancerDao.listByNetworkIdAndScheme(guestNetworkId, Scheme.Public); - final List lbRules = new ArrayList(); + final List lbRules = new ArrayList<>(); if (_networkModel.isProviderSupportServiceInNetwork(guestNetworkId, Service.Lb, provider)) { // Re-apply load balancing rules createLoadBalancingRulesList(lbRules, lbs); @@ -2695,18 +2632,11 @@ Configurable, StateListener rules, final long networkId) { final NetworkVO network = _networkDao.findById(networkId); final NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); - final Boolean defaultEgressPolicy = offering.isEgressDefaultPolicy(); + final boolean defaultEgressPolicy = offering.isEgressDefaultPolicy(); // The default on the router is set to Deny all. So, if the default configuration in the offering is set to true (Allow), we change the Egress here if (defaultEgressPolicy) { - final List sourceCidr = new ArrayList(); - final List destCidr = new ArrayList(); - - sourceCidr.add(network.getCidr()); - destCidr.add(NetUtils.ALL_IP4_CIDRS); - - final FirewallRule rule = new FirewallRuleVO(null, null, null, null, NetUtils.ALL_PROTO, networkId, network.getAccountId(), network.getDomainId(), Purpose.Firewall, sourceCidr, - destCidr, null, null, null, FirewallRule.TrafficType.Egress, FirewallRule.FirewallRuleType.System); + final FirewallRule rule = getFirewallRule(network.getCidr(), NetUtils.ALL_IP4_CIDRS, networkId, network, Purpose.Firewall); rules.add(rule); } else { @@ -2714,18 +2644,24 @@ Configurable, StateListener sourceCidr = new ArrayList<>(); + final List destCidr = new ArrayList<>(); + + sourceCidr.add(cidr); + destCidr.add(allIp4Cidrs); + + return new FirewallRuleVO(null, null, null, null, NetUtils.ALL_PROTO, networkId, network.getAccountId(), network.getDomainId(), firewall, sourceCidr, + destCidr, null, null, null, FirewallRule.TrafficType.Egress, FirewallRule.FirewallRuleType.System); + } + private void createDefaultEgressIpv6FirewallRule(final List rules, final long networkId) { final NetworkVO network = _networkDao.findById(networkId); if(!_networkOfferingDao.isIpv6Supported(network.getNetworkOfferingId())) { return; } // Since not all networks will IPv6 supported, add a system rule for IPv6 networks - final List sourceCidr = new ArrayList(); - final List destCidr = new ArrayList(); - sourceCidr.add(network.getIp6Cidr()); - destCidr.add(NetUtils.ALL_IP6_CIDRS); - final FirewallRule rule = new FirewallRuleVO(null, null, null, null, NetUtils.ALL_PROTO, networkId, network.getAccountId(), network.getDomainId(), Purpose.Ipv6Firewall, sourceCidr, - destCidr, null, null, null, FirewallRule.TrafficType.Egress, FirewallRule.FirewallRuleType.System); + final FirewallRule rule = getFirewallRule(network.getIp6Cidr(), NetUtils.ALL_IP6_CIDRS, networkId, network, Purpose.Ipv6Firewall); rules.add(rule); } @@ -2754,7 +2690,7 @@ Configurable, StateListener userIps = _networkModel.listPublicIpsAssignedToGuestNtwk(guestNetworkId, null); - final List allPublicIps = new ArrayList(); + final List allPublicIps = new ArrayList<>(); if (userIps != null && !userIps.isEmpty()) { boolean addIp = true; for (final IpAddress userIp : userIps) { @@ -2783,8 +2719,7 @@ Configurable, StateListener publicIps = providerToIpList.get(provider); - return publicIps; + return providerToIpList.get(provider); } @Override @@ -2797,7 +2732,7 @@ Configurable, StateListener guestNetworks = new ArrayList(); + final List guestNetworks = new ArrayList<>(); final GetDomRVersionAnswer versionAnswer = (GetDomRVersionAnswer) cmds.getAnswer("getDomRVersion"); router.setTemplateVersion(versionAnswer.getTemplateVersion()); @@ -2959,7 +2894,6 @@ Configurable, StateListener revokedIpAliasVOs = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.State.revoked); logger.debug("Found" + revokedIpAliasVOs.size() + "ip Aliases to revoke on the router as a part of dhcp configuration"); - final List revokedIpAliasTOs = new ArrayList(); + final List revokedIpAliasTOs = new ArrayList<>(); for (final NicIpAliasVO revokedAliasVO : revokedIpAliasVOs) { revokedIpAliasTOs.add(new IpAliasTO(revokedAliasVO.getIp4Address(), revokedAliasVO.getNetmask(), revokedAliasVO.getAliasCount().toString())); } final List aliasVOs = _nicIpAliasDao.listByNetworkIdAndState(network.getId(), NicIpAlias.State.active); logger.debug("Found" + aliasVOs.size() + "ip Aliases to apply on the router as a part of dhcp configuration"); - final List activeIpAliasTOs = new ArrayList(); + final List activeIpAliasTOs = new ArrayList<>(); for (final NicIpAliasVO aliasVO : aliasVOs) { activeIpAliasTOs.add(new IpAliasTO(aliasVO.getIp4Address(), aliasVO.getNetmask(), aliasVO.getAliasCount().toString())); } @@ -3046,9 +2980,6 @@ Configurable, StateListener params = new HashMap(); + final Map params = new HashMap<>(); if (reprogramNetwork) { params.put(Param.ReProgramGuestNetworks, true); } else { @@ -3105,10 +3036,8 @@ Configurable, StateListener getRoutersForNetwork(final long networkId) { final List routers = _routerDao.findByNetwork(networkId); - final List vrs = new ArrayList(routers.size()); - for (final DomainRouterVO router : routers) { - vrs.add(router); - } + final List vrs = new ArrayList<>(routers.size()); + vrs.addAll(routers); return vrs; } @@ -3151,16 +3080,13 @@ Configurable, StateListener routerNics = new ArrayList(); + List routerNics = new ArrayList<>(); if (nic != null) { routerNics.add(nic); } else { @@ -3232,7 +3158,7 @@ Configurable, StateListener answerFinal.getBytesReceived()) { - if (logger.isDebugEnabled()) { - logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesReceived()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesReceived())); - } + logger.debug("Received # of bytes that's less than the last one. Assuming something went wrong and persisting it. Router: {} Reported: {} Stored: {}" + , answerFinal.getRouterName() + , toHumanReadableSize(answerFinal.getBytesReceived()) + , toHumanReadableSize(stats.getCurrentBytesReceived())); stats.setNetBytesReceived(stats.getNetBytesReceived() + stats.getCurrentBytesReceived()); } stats.setCurrentBytesReceived(answerFinal.getBytesReceived()); if (stats.getCurrentBytesSent() > answerFinal.getBytesSent()) { - if (logger.isDebugEnabled()) { - logger.debug("Received # of bytes that's less than the last one. " + "Assuming something went wrong and persisting it. Router: " - + answerFinal.getRouterName() + " Reported: " + toHumanReadableSize(answerFinal.getBytesSent()) + " Stored: " + toHumanReadableSize(stats.getCurrentBytesSent())); - } + logger.debug("Received # of bytes that's less than the last one. Assuming something went wrong and persisting it. Router: {} Reported: {} Stored: {}" + , answerFinal.getRouterName() + , toHumanReadableSize(answerFinal.getBytesReceived()) + , toHumanReadableSize(stats.getCurrentBytesReceived())); stats.setNetBytesSent(stats.getNetBytesSent() + stats.getCurrentBytesSent()); } stats.setCurrentBytesSent(answerFinal.getBytesSent()); @@ -3316,7 +3242,7 @@ Configurable, StateListener upgradeRouterTemplate(final UpgradeRouterTemplateCmd cmd) { - List routers = new ArrayList(); + List routers = new ArrayList<>(); int params = 0; final Long routerId = cmd.getId(); @@ -3375,11 +3301,11 @@ Configurable, StateListener rebootRouters(final List routers) { - final List jobIds = new ArrayList(); + final List jobIds = new ArrayList<>(); for (final DomainRouterVO router : routers) { if (!_nwHelper.checkRouterTemplateVersion(router)) { logger.debug("Upgrading template for router: {}", router); - final Map params = new HashMap(); + final Map params = new HashMap<>(); params.put("ctxUserId", "1"); params.put("ctxAccountId", "" + router.getAccountId()); @@ -3455,30 +3381,28 @@ Configurable, StateListener - if (opaque != null && opaque instanceof Pair) { + if (opaque instanceof Pair) { final Pair pair = (Pair)opaque; final Object first = pair.first(); final Object second = pair.second(); // powerHostId cannot be null in case of out-of-band VM movement - if (second != null && second instanceof Long) { + if (second instanceof Long) { final Long powerHostId = (Long)second; Long hostId = null; - if (first != null && first instanceof Long) { + if (first instanceof Long) { hostId = (Long)first; } // The following scenarios are due to out-of-band VM movement // 1. If VM is in stopped state in CS due to 'PowerMissing' report from old host (hostId is null) and then there is a 'PowerOn' report from new host // 2. If VM is in running state in CS and there is a 'PowerOn' report from new host - if (hostId == null || hostId.longValue() != powerHostId.longValue()) { - return true; - } + return hostId == null || hostId.longValue() != powerHostId.longValue(); } } return false; } protected boolean aggregationExecution(final AggregationControlCommand.Action action, final Network network, final List routers) - throws AgentUnavailableException, ResourceUnavailableException { + throws ResourceUnavailableException { int errors = 0; @@ -3508,12 +3432,12 @@ Configurable, StateListener routers) throws AgentUnavailableException, ResourceUnavailableException { + public boolean prepareAggregatedExecution(final Network network, final List routers) throws ResourceUnavailableException { return aggregationExecution(Action.Start, network, routers); } @Override - public boolean completeAggregatedExecution(final Network network, final List routers) throws AgentUnavailableException, ResourceUnavailableException { + public boolean completeAggregatedExecution(final Network network, final List routers) throws ResourceUnavailableException { return aggregationExecution(Action.Finish, network, routers); } } diff --git a/systemvm/debian/root/health_checks/cpu_usage_check.py b/systemvm/debian/root/health_checks/cpu_usage_check.py index ab2c4f1c46e..270ea208610 100644 --- a/systemvm/debian/root/health_checks/cpu_usage_check.py +++ b/systemvm/debian/root/health_checks/cpu_usage_check.py @@ -29,7 +29,7 @@ def main(): if "maxCpuUsage" not in data: print("Missing maxCpuUsage in health_checks_data systemThresholds, skipping") - exit(0) + exit(3) maxCpuUsage = float(data["maxCpuUsage"]) cmd = "top -b -n2 -p 1 | fgrep \"Cpu(s)\" | tail -1 | " \ @@ -48,7 +48,7 @@ def main(): exit(0) else: print("Failed to retrieve cpu usage using " + cmd) - exit(1) + exit(3) if __name__ == "__main__": diff --git a/systemvm/debian/root/health_checks/dhcp_check.py b/systemvm/debian/root/health_checks/dhcp_check.py index 025e494c247..0e03f4a798a 100755 --- a/systemvm/debian/root/health_checks/dhcp_check.py +++ b/systemvm/debian/root/health_checks/dhcp_check.py @@ -25,7 +25,7 @@ def main(): if vMs is None or len(vMs) == 0: print("No VMs running data available, skipping") - exit(0) + exit(3) try: with open('/etc/dhcphosts.txt', 'r') as hostsFile: diff --git a/systemvm/debian/root/health_checks/disk_space_check.py b/systemvm/debian/root/health_checks/disk_space_check.py index f6c9a7fc497..3f2d28241bb 100644 --- a/systemvm/debian/root/health_checks/disk_space_check.py +++ b/systemvm/debian/root/health_checks/disk_space_check.py @@ -28,7 +28,7 @@ def main(): if "minDiskNeeded" not in data: print("Missing minDiskNeeded in health_checks_data systemThresholds, skipping") - exit(0) + exit(3) minDiskNeeded = float(data["minDiskNeeded"]) * 1024 s = statvfs('/') diff --git a/systemvm/debian/root/monitorServices.py b/systemvm/debian/root/monitorServices.py index 0b7c9db8a5a..1db0e15dbfc 100755 --- a/systemvm/debian/root/monitorServices.py +++ b/systemvm/debian/root/monitorServices.py @@ -34,6 +34,13 @@ class StatusCodes: STOPPED = 4 STARTING = 5 +# see com.cloud.network.VirtualNetworkApplianceService.RouterHealthStatus and make sure to keep it aligned +class RouterHealthStatus: + SUCCESS = "SUCCESS" + FAILED = "FAILED" + WARNING = "WARNING" + UNKNOWN = "UNKNOWN" + class Log: INFO = 'INFO' ALERT = 'ALERT' @@ -267,24 +274,35 @@ def monitProcess( processes_info ): printd ("---------------------------\nchecking the service %s\n---------------------------- " %process) serviceName = process + ".service" processStatus, wasRestarted = checkProcessStatus(properties) - if processStatus != StatusCodes.RUNNING: - printd( "\n Service %s is not Running"%process) - checkEndTime = time.time() - service_status[serviceName] = { - "success": "false", - "lastUpdate": str(int(checkStartTime * 1000)), - "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), - "message": "service down at last check " + str(csec) - } + routerHealth = RouterHealthStatus.UNKNOWN + + match processStatus: + case StatusCodes.RUNNING: + routerHealth = RouterHealthStatus.SUCCESS + routerMessage = "service is running" + (", was restarted" if wasRestarted else "") + case StatusCodes.STARTING: + routerHealth = RouterHealthStatus.WARNING + routerMessage = "service is starting at " + str(csec) + case StatusCodes.STOPPED: + routerHealth = RouterHealthStatus.WARNING + routerMessage = "service down at last check " + str(csec) + case StatusCodes.SUCCESS: + routerHealth = RouterHealthStatus.UNKNOWN + routerMessage = "service exisits but no status" + case StatusCodes.FAILED | StatusCodes.INVALID_INP: + routerHealth = RouterHealthStatus.FAILED + routerMessage = "service down at last check " + str(csec) + + printd( "\n Service %s is status == " % routerHealth) + checkEndTime = time.time() + service_status[serviceName] = { + "success": routerHealth, + "lastUpdate": str(int(checkStartTime * 1000)), + "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), + "message": routerMessage + } + if routerHealth != RouterHealthStatus.SUCCESS: failing_services.append(serviceName) - else: - checkEndTime = time.time() - service_status[serviceName] = { - "success": "true", - "lastUpdate": str(int(checkStartTime * 1000)), - "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), - "message": "service is running" + (", was restarted" if wasRestarted else "") - } return service_status, failing_services @@ -299,24 +317,26 @@ def execute(script, checkType = "basic"): output = pout.communicate()[0].decode().strip() checkEndTime = time.time() - if exitStatus == 0: - if len(output) > 0: - printd("Successful execution of " + script) - return { - "success": "true", - "lastUpdate": str(int(checkStartTime * 1000)), - "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), - "message": output - } - return {} #Skip script if no output is received - else: - printd("Script execution failed " + script) - return { - "success": "false", - "lastUpdate": str(int(checkStartTime * 1000)), - "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), - "message": output - } + # we run all scripts and have to ignore the ones that do nothing + if not len(output) > 0 and exitStatus == 0: + return {} + + routerHealth = RouterHealthStatus.SUCCESS + match exitStatus: + case 1: + routerHealth = RouterHealthStatus.FAILED + case 2: + routerHealth = RouterHealthStatus.WARNING + case 3: + routerHealth = RouterHealthStatus.UNKNOWN + + printd("Ended execution of " + script) + return { + "success": routerHealth, + "lastUpdate": str(int(checkStartTime * 1000)), + "lastRunDuration": str((checkEndTime - checkStartTime) * 1000), + "message": output + } def main(checkType = "basic"): startTime = time.time() @@ -349,7 +369,7 @@ def main(checkType = "basic"): ret = execute(fpath, checkType) if len(ret) == 0: continue - if "success" in ret and ret["success"].lower() == "false": + if "success" in ret and ret["success"].upper() == RouterHealthStatus.FAILED: failingChecks.append(f) monitResult[f] = ret diff --git a/ui/src/views/infra/routers/RouterHealthCheck.vue b/ui/src/views/infra/routers/RouterHealthCheck.vue index fbe3b658fa9..89a05d1fb94 100644 --- a/ui/src/views/infra/routers/RouterHealthCheck.vue +++ b/ui/src/views/infra/routers/RouterHealthCheck.vue @@ -35,7 +35,7 @@ size="large"> @@ -113,6 +113,7 @@ export default { }, { key: 'status', + dataIndex: 'status', title: this.$t('label.router.health.check.success') }, { From 393b5d2b7791a9ab280337047fd577a609cbd2b7 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 22 Sep 2025 10:25:21 +0200 Subject: [PATCH 011/145] VR: consider NICs for remote access VPN when apply dhcp entry (#11681) --- systemvm/debian/opt/cloud/bin/cs/CsDhcp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index 35849e4c5f8..574a352b5f3 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -226,7 +226,7 @@ class CsDhcp(CsDataBag): i = IPAddress(entry['ipv4_address']) # Calculate the device for v in self.devinfo: - if i > v['network'].network and i < v['network'].broadcast: + if i > v['network'].network and v['network'].broadcast and i < v['network'].broadcast: v['dnsmasq'] = True # Virtual Router v['gateway'] = entry['default_gateway'] From 69998f7c40fb0179b2ab5edcf50927c0e9f1fc17 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Mon, 22 Sep 2025 10:46:43 +0200 Subject: [PATCH 012/145] api,server: support templatetype when upload template from local (#11682) --- .../GetUploadParamsForTemplateCmd.java | 9 ++++++ .../upload/params/TemplateUploadParams.java | 5 ++-- .../storage/upload/params/UploadParams.java | 2 ++ .../upload/params/UploadParamsBase.java | 28 ++++++++++++++----- .../cloud/template/TemplateAdapterBase.java | 16 +++++++---- .../cloud/template/TemplateManagerImpl.java | 5 +++- 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java index 330224a6055..2472a03b890 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java @@ -104,6 +104,11 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { description = "if true, the templates would be available for deploying CKS clusters", since = "4.21.0") protected Boolean forCks; + @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING, + description = "the type of the template. Valid options are: USER/VNF (for all users) and SYSTEM/ROUTING/BUILTIN (for admins only).", + since = "4.22.0") + private String templateType; + public String getDisplayText() { return StringUtils.isBlank(displayText) ? getName() : displayText; } @@ -181,6 +186,10 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { return CPU.CPUArch.fromType(arch); } + public String getTemplateType() { + return templateType; + } + @Override public void execute() throws ServerApiException { validateRequest(); diff --git a/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java b/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java index 769aa3dc1f2..aae7c0be812 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java +++ b/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java @@ -30,10 +30,11 @@ public class TemplateUploadParams extends UploadParamsBase { Long zoneId, Hypervisor.HypervisorType hypervisorType, String chksum, String templateTag, long templateOwnerId, Map details, Boolean sshkeyEnabled, - Boolean isDynamicallyScalable, Boolean isRoutingType, boolean deployAsIs, boolean forCks) { + Boolean isDynamicallyScalable, Boolean isRoutingType, boolean deployAsIs, + boolean forCks, String templateType) { super(userId, name, displayText, arch, bits, passwordEnabled, requiresHVM, isPublic, featured, isExtractable, format, guestOSId, zoneId, hypervisorType, chksum, templateTag, templateOwnerId, details, - sshkeyEnabled, isDynamicallyScalable, isRoutingType, deployAsIs, forCks); + sshkeyEnabled, isDynamicallyScalable, isRoutingType, deployAsIs, forCks, templateType); setBootable(true); } } diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java index 7be526d780d..5738cfaad88 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParams.java @@ -49,4 +49,6 @@ public interface UploadParams { boolean isDirectDownload(); boolean isDeployAsIs(); CPU.CPUArch getArch(); + boolean isForCks(); + String getTemplateType(); } diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java index c3499d75c3b..945551a9318 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java @@ -48,15 +48,17 @@ public abstract class UploadParamsBase implements UploadParams { private boolean deployAsIs; private boolean forCks; private CPU.CPUArch arch; + private String templateType; UploadParamsBase(long userId, String name, String displayText, CPU.CPUArch arch, - Integer bits, boolean passwordEnabled, boolean requiresHVM, - boolean isPublic, boolean featured, - boolean isExtractable, String format, Long guestOSId, - Long zoneId, Hypervisor.HypervisorType hypervisorType, String checksum, - String templateTag, long templateOwnerId, - Map details, boolean sshkeyEnabled, - boolean isDynamicallyScalable, boolean isRoutingType, boolean deployAsIs, boolean forCks) { + Integer bits, boolean passwordEnabled, boolean requiresHVM, + boolean isPublic, boolean featured, + boolean isExtractable, String format, Long guestOSId, + Long zoneId, Hypervisor.HypervisorType hypervisorType, String checksum, + String templateTag, long templateOwnerId, + Map details, boolean sshkeyEnabled, + boolean isDynamicallyScalable, boolean isRoutingType, boolean deployAsIs, + boolean forCks, String templateType) { this.userId = userId; this.name = name; this.displayText = displayText; @@ -79,6 +81,8 @@ public abstract class UploadParamsBase implements UploadParams { this.isDynamicallyScalable = isDynamicallyScalable; this.isRoutingType = isRoutingType; this.deployAsIs = deployAsIs; + this.forCks = forCks; + this.templateType = templateType; } UploadParamsBase(long userId, String name, String displayText, boolean isPublic, boolean isFeatured, @@ -261,4 +265,14 @@ public abstract class UploadParamsBase implements UploadParams { public void setArch(CPU.CPUArch arch) { this.arch = arch; } + + @Override + public boolean isForCks() { + return forCks; + } + + @Override + public String getTemplateType() { + return templateType; + } } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 75c63aebe0d..a9adfa93236 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -29,6 +29,7 @@ import java.util.Set; import javax.inject.Inject; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.GetUploadParamsForIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; @@ -469,7 +470,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat /** * Prepare upload parameters internal method for templates and ISOs local upload */ - private TemplateProfile prepareUploadParamsInternal(UploadParams params) throws ResourceAllocationException { + private TemplateProfile prepareUploadParamsInternal(BaseCmd cmd, UploadParams params) throws ResourceAllocationException { //check if the caller can operate with the template owner Account caller = CallContext.current().getCallingAccount(); Account owner = _accountMgr.getAccount(params.getTemplateOwnerId()); @@ -490,12 +491,16 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat StringUtils.join(Arrays.stream(HypervisorType.values()).filter(h -> h != HypervisorType.None).map(HypervisorType::name).toArray(), ", "))); } + TemplateType templateType = templateMgr.validateTemplateType(cmd, _accountMgr.isAdmin(caller.getAccountId()), + false, params.getHypervisorType()); + return prepare(params.isIso(), params.getUserId(), params.getName(), params.getDisplayText(), params.getArch(), params.getBits(), params.isPasswordEnabled(), params.requiresHVM(), params.getUrl(), params.isPublic(), params.isFeatured(), params.isExtractable(), params.getFormat(), params.getGuestOSId(), zoneList, params.getHypervisorType(), params.getChecksum(), params.isBootable(), params.getTemplateTag(), owner, params.getDetails(), params.isSshKeyEnabled(), params.getImageStoreUuid(), - params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload(), params.isDeployAsIs(), false, null); + params.isDynamicallyScalable(), templateType, params.isDirectDownload(), params.isDeployAsIs(), + params.isForCks(), null); } private Long getDefaultDeployAsIsGuestOsId() { @@ -516,8 +521,9 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat BooleanUtils.toBoolean(cmd.isFeatured()), BooleanUtils.toBoolean(cmd.isExtractable()), cmd.getFormat(), osTypeId, cmd.getZoneId(), HypervisorType.getType(cmd.getHypervisor()), cmd.getChecksum(), cmd.getTemplateTag(), cmd.getEntityOwnerId(), cmd.getDetails(), BooleanUtils.toBoolean(cmd.isSshKeyEnabled()), - BooleanUtils.toBoolean(cmd.isDynamicallyScalable()), BooleanUtils.toBoolean(cmd.isRoutingType()), cmd.isDeployAsIs(), cmd.isForCks()); - return prepareUploadParamsInternal(params); + BooleanUtils.toBoolean(cmd.isDynamicallyScalable()), BooleanUtils.toBoolean(cmd.isRoutingType()), cmd.isDeployAsIs(), + cmd.isForCks(), cmd.getTemplateType()); + return prepareUploadParamsInternal(cmd, params); } @Override @@ -526,7 +532,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat cmd.getDisplayText(), BooleanUtils.toBoolean(cmd.isPublic()), BooleanUtils.toBoolean(cmd.isFeatured()), BooleanUtils.toBoolean(cmd.isExtractable()), cmd.getOsTypeId(), cmd.getZoneId(), BooleanUtils.toBoolean(cmd.isBootable()), cmd.getEntityOwnerId()); - return prepareUploadParamsInternal(params); + return prepareUploadParamsInternal(cmd, params); } @Override diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 5518a29955b..7bb66958cb4 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -2339,7 +2339,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public TemplateType validateTemplateType(BaseCmd cmd, boolean isAdmin, boolean isCrossZones, HypervisorType hypervisorType) { - if (!(cmd instanceof UpdateTemplateCmd) && !(cmd instanceof RegisterTemplateCmd)) { + if (!(cmd instanceof UpdateTemplateCmd) && !(cmd instanceof RegisterTemplateCmd) && !(cmd instanceof GetUploadParamsForTemplateCmd)) { return null; } TemplateType templateType = null; @@ -2351,6 +2351,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } else if (cmd instanceof RegisterTemplateCmd) { newType = ((RegisterTemplateCmd)cmd).getTemplateType(); isRoutingType = ((RegisterTemplateCmd)cmd).isRoutingType(); + } else if (cmd instanceof GetUploadParamsForTemplateCmd) { + newType = ((GetUploadParamsForTemplateCmd)cmd).getTemplateType(); + isRoutingType = ((GetUploadParamsForTemplateCmd)cmd).isRoutingType(); } if (newType != null) { try { From 12513e18fa3f77752d5fefc833561e8e44034920 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Mon, 22 Sep 2025 15:52:50 +0530 Subject: [PATCH 013/145] server: Update gson date format for serializing/deserializing Date in MS stats (#11506) * Update gson date format for serializing/deserializing Date in MS stats (across multiple management servers) * review * review comments, and unit tests * added unit test with different date format * Use separate Gson for MS stats serialization/deserialization --- .../ManagementServerHostStatsEntry.java | 2 +- .../java/com/cloud/server/StatsCollector.java | 13 +- .../com/cloud/server/StatsCollectorTest.java | 112 ++++++++++++++++++ .../main/java/com/cloud/utils/DateUtil.java | 2 +- 4 files changed, 122 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/com/cloud/server/ManagementServerHostStatsEntry.java b/server/src/main/java/com/cloud/server/ManagementServerHostStatsEntry.java index 172ab1e83eb..08cc54f9799 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerHostStatsEntry.java +++ b/server/src/main/java/com/cloud/server/ManagementServerHostStatsEntry.java @@ -105,7 +105,7 @@ public class ManagementServerHostStatsEntry implements ManagementServerHostStats } @Override - public Date getCollectionTime(){ + public Date getCollectionTime() { return collectionTime; } diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java b/server/src/main/java/com/cloud/server/StatsCollector.java index 4e1e78f1610..7e83d452bb9 100644 --- a/server/src/main/java/com/cloud/server/StatsCollector.java +++ b/server/src/main/java/com/cloud/server/StatsCollector.java @@ -46,6 +46,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; @@ -170,10 +171,10 @@ import com.codahale.metrics.jvm.GarbageCollectorMetricSet; import com.codahale.metrics.jvm.MemoryUsageGaugeSet; import com.codahale.metrics.jvm.ThreadStatesGaugeSet; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; -import com.google.gson.reflect.TypeToken; import com.sun.management.OperatingSystemMXBean; /** @@ -294,6 +295,9 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc private static StatsCollector s_instance = null; private static Gson gson = new Gson(); + private static Gson msStatsGson = new GsonBuilder() + .setDateFormat(DateUtil.ZONED_DATETIME_FORMAT) + .create(); private ScheduledExecutorService _executor = null; @Inject @@ -739,7 +743,6 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc dbStats.put(uptime, (Long.valueOf(stats.get(uptime)))); } - @Override protected Point createInfluxDbPoint(Object metricsObject) { return null; @@ -759,7 +762,7 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc hostStatsEntry = getDataFrom(mshost); managementServerHostStats.put(mshost.getUuid(), hostStatsEntry); // send to other hosts - clusterManager.publishStatus(gson.toJson(hostStatsEntry)); + clusterManager.publishStatus(msStatsGson.toJson(hostStatsEntry)); } catch (Throwable t) { // pokemon catch to make sure the thread stays running logger.error("Error trying to retrieve management server host statistics", t); @@ -1158,9 +1161,9 @@ public class StatsCollector extends ManagerBase implements ComponentMethodInterc logger.debug(String.format("StatusUpdate from %s, json: %s", pdu.getSourcePeer(), pdu.getJsonPackage())); } - ManagementServerHostStatsEntry hostStatsEntry = null; + ManagementServerHostStatsEntry hostStatsEntry; try { - hostStatsEntry = gson.fromJson(pdu.getJsonPackage(),new TypeToken(){}.getType()); + hostStatsEntry = msStatsGson.fromJson(pdu.getJsonPackage(), ManagementServerHostStatsEntry.class); managementServerHostStats.put(hostStatsEntry.getManagementServerHostUuid(), hostStatsEntry); // Update peer state to Up in mshost_peer diff --git a/server/src/test/java/com/cloud/server/StatsCollectorTest.java b/server/src/test/java/com/cloud/server/StatsCollectorTest.java index 6a979259cd9..3578e6948a4 100644 --- a/server/src/test/java/com/cloud/server/StatsCollectorTest.java +++ b/server/src/test/java/com/cloud/server/StatsCollectorTest.java @@ -20,8 +20,10 @@ package com.cloud.server; import static org.mockito.Mockito.when; +import java.lang.reflect.Field; import java.net.URI; import java.net.URISyntaxException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Date; @@ -33,6 +35,8 @@ import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import com.cloud.utils.DateUtil; +import com.google.gson.JsonSyntaxException; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; @@ -116,6 +120,8 @@ public class StatsCollectorTest { private static Gson gson = new Gson(); + private Gson msStatsGson; + private MockedStatic influxDBFactoryMocked; private AutoCloseable closeable; @@ -125,6 +131,9 @@ public class StatsCollectorTest { closeable = MockitoAnnotations.openMocks(this); statsCollector.vmStatsDao = vmStatsDaoMock; statsCollector.volumeStatsDao = volumeStatsDao; + Field msStatsGsonField = StatsCollector.class.getDeclaredField("msStatsGson"); + msStatsGsonField.setAccessible(true); + msStatsGson = (Gson) msStatsGsonField.get(null); } @After @@ -612,4 +621,107 @@ public class StatsCollectorTest { Mockito.verify(mockPool, Mockito.never()).setCapacityIops(Mockito.anyLong()); Mockito.verify(mockPool, Mockito.never()).setUsedIops(Mockito.anyLong()); } + + @Test + public void testGsonDateFormatSerialization() { + Date now = new Date(); + TestClass testObj = new TestClass("TestString", 999, now); + String json = msStatsGson.toJson(testObj); + + Assert.assertTrue(json.contains("TestString")); + Assert.assertTrue(json.contains("999")); + String expectedDate = new SimpleDateFormat(DateUtil.ZONED_DATETIME_FORMAT).format(now); + Assert.assertTrue(json.contains(expectedDate)); + } + + @Test + public void testGsonDateFormatDeserializationWithSameDateFormat() throws Exception { + String json = "{\"str\":\"TestString\",\"num\":999,\"date\":\"2025-08-22T15:39:43+0000\"}"; + TestClass testObj = msStatsGson.fromJson(json, TestClass.class); + + Assert.assertEquals("TestString", testObj.getStr()); + Assert.assertEquals(999, testObj.getNum()); + Date expectedDate = new SimpleDateFormat(DateUtil.ZONED_DATETIME_FORMAT).parse("2025-08-22T15:39:43+0000"); + Assert.assertEquals(expectedDate, testObj.getDate()); + } + + @Test (expected = JsonSyntaxException.class) + public void testGsonDateFormatDeserializationWithDifferentDateFormat() throws Exception { + String json = "{\"str\":\"TestString\",\"num\":999,\"date\":\"22/08/2025T15:39:43+0000\"}"; + msStatsGson.fromJson(json, TestClass.class); + /* Deserialization throws the below exception: + com.google.gson.JsonSyntaxException: 22/08/2025T15:39:43+0000 + at com.google.gson.DefaultTypeAdapters$DefaultDateTypeAdapter.deserializeToDate(DefaultTypeAdapters.java:376) + at com.google.gson.DefaultTypeAdapters$DefaultDateTypeAdapter.deserialize(DefaultTypeAdapters.java:351) + at com.google.gson.DefaultTypeAdapters$DefaultDateTypeAdapter.deserialize(DefaultTypeAdapters.java:307) + at com.google.gson.JsonDeserializationVisitor.invokeCustomDeserializer(JsonDeserializationVisitor.java:92) + at com.google.gson.JsonObjectDeserializationVisitor.visitFieldUsingCustomHandler(JsonObjectDeserializationVisitor.java:117) + at com.google.gson.ReflectingFieldNavigator.visitFieldsReflectively(ReflectingFieldNavigator.java:63) + at com.google.gson.ObjectNavigator.accept(ObjectNavigator.java:120) + at com.google.gson.JsonDeserializationContextDefault.fromJsonObject(JsonDeserializationContextDefault.java:76) + at com.google.gson.JsonDeserializationContextDefault.deserialize(JsonDeserializationContextDefault.java:54) + at com.google.gson.Gson.fromJson(Gson.java:551) + at com.google.gson.Gson.fromJson(Gson.java:498) + at com.google.gson.Gson.fromJson(Gson.java:467) + at com.google.gson.Gson.fromJson(Gson.java:417) + at com.google.gson.Gson.fromJson(Gson.java:389) + at com.cloud.serializer.GsonHelperTest.testGsonDateFormatDeserializationWithDifferentDateFormat(GsonHelperTest.java:113) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) + at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) + at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) + at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) + at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) + at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) + at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) + at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) + at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) + at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) + at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) + at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) + at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) + at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) + at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) + at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) + at org.junit.runners.ParentRunner.run(ParentRunner.java:413) + at org.junit.runner.JUnitCore.run(JUnitCore.java:137) + at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:69) + at com.intellij.rt.junit.IdeaTestRunner$Repeater$1.execute(IdeaTestRunner.java:38) + at com.intellij.rt.execution.junit.TestsRepeater.repeat(TestsRepeater.java:11) + at com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:35) + at com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:231) + at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:55) + Caused by: java.text.ParseException: Unparseable date: "22/08/2025T15:39:43+0000" + at java.base/java.text.DateFormat.parse(DateFormat.java:395) + at com.google.gson.DefaultTypeAdapters$DefaultDateTypeAdapter.deserializeToDate(DefaultTypeAdapters.java:374) + ... 42 more + */ + } + + private static class TestClass { + private String str; + private int num; + private Date date; + + public TestClass(String str, int num, Date date) { + this.str = str; + this.num = num; + this.date = date; + } + + public String getStr() { + return str; + } + + public int getNum() { + return num; + } + + public Date getDate() { + return date; + } + } } diff --git a/utils/src/main/java/com/cloud/utils/DateUtil.java b/utils/src/main/java/com/cloud/utils/DateUtil.java index fdf2ba8fe01..00ae5565dad 100644 --- a/utils/src/main/java/com/cloud/utils/DateUtil.java +++ b/utils/src/main/java/com/cloud/utils/DateUtil.java @@ -48,7 +48,7 @@ public class DateUtil { public static final TimeZone GMT_TIMEZONE = TimeZone.getTimeZone("GMT"); public static final String YYYYMMDD_FORMAT = "yyyyMMddHHmmss"; - private static final String ZONED_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ"; + public static final String ZONED_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ"; private static final DateFormat ZONED_DATETIME_SIMPLE_FORMATTER = new SimpleDateFormat(ZONED_DATETIME_FORMAT); private static final DateTimeFormatter[] parseFormats = new DateTimeFormatter[]{ From d9abc078cfde6c406d193456de11866e6859441f Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 23 Sep 2025 09:37:43 +0530 Subject: [PATCH 014/145] ui: fix build on latest Ubuntu and macOS (#11259) * ui: fix build on latest Ubuntu and macOS This fixes UI build, so it can build against nodejs 20.x/LTS. Signed-off-by: Rohit Yadav * Update ui/package.json --------- Signed-off-by: Rohit Yadav --- ui/README.md | 10 +++++++--- ui/package.json | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ui/README.md b/ui/README.md index 6e11b318c93..c794c22b182 100644 --- a/ui/README.md +++ b/ui/README.md @@ -8,18 +8,18 @@ A modern role-based progressive CloudStack UI based on Vue.js and Ant Design. Install node: (Debian/Ubuntu) - curl -sL https://deb.nodesource.com/setup_16.x | sudo -E bash - + curl -sL https://deb.nodesource.com/setup_20.x | sudo -E bash - sudo apt-get install -y nodejs # Or use distro provided: sudo apt-get install npm nodejs Install node: (CentOS/Fedora/RHEL) - curl -sL https://rpm.nodesource.com/setup_16.x | sudo bash - + curl -sL https://rpm.nodesource.com/setup_20.x | sudo bash - sudo yum install nodejs Install node: (Mac OS) - brew install node@16 + brew install node@20 Optionally, you may also install system-wide dev tools: @@ -44,6 +44,10 @@ To use the community Simulator QA server you can do this: echo "CS_URL=https://qa.cloudstack.cloud" > .env.local +This may be required for newer npm versions: + + export NODE_OPTIONS=--openssl-legacy-provider + Build and run: npm run serve diff --git a/ui/package.json b/ui/package.json index 3c7e1299a9d..48f337500bd 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,7 +1,7 @@ { "name": "cloudstack-ui", "description": "Modern role-based Apache CloudStack UI", - "version": "4.19.0", + "version": "4.22.0", "homepage": "https://cloudstack.apache.org/", "repository": { "type": "git", @@ -106,8 +106,10 @@ "uglifyjs-webpack-plugin": "^2.2.0", "vue-jest": "^5.0.0-0", "vue-svg-loader": "^0.17.0-beta.2", - "webpack": "^4.46.0" + "webpack": "^4.46.0", + "node-gyp": "10.0.1", "nan": "2.18.0" }, + "resolutions": { "nan": "2.18.0" }, "eslintConfig": { "root": true, "env": { From 40dec996591a99d40b51313fa23c8d937bed780f Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Tue, 23 Sep 2025 12:07:10 +0530 Subject: [PATCH 015/145] server: Cleanup allocated snapshots / vm snapshots, and update pending ones to Error on MS start (#8452) * Remove allocated snapshots / vm snapshots on start * Check and Cleanup snapshots / vm snapshots on MS start * rebase fixes * Update volume state (from Snapshotting) on MS start when its snapshot job not finished and snapshot in Creating state --- .../cloud/vm/snapshot/VMSnapshotService.java | 3 + .../api/storage/SnapshotDataFactory.java | 3 + .../api/storage/VMSnapshotStrategy.java | 3 + .../cloud/vm/snapshot/VMSnapshotManager.java | 1 - .../cloud/vm/snapshot/dao/VMSnapshotDao.java | 2 + .../snapshot/SnapshotDataFactoryImpl.java | 22 +++++++- .../vmsnapshot/DefaultVMSnapshotStrategy.java | 10 ++++ .../vmsnapshot/ScaleIOVMSnapshotStrategy.java | 10 ++++ .../jobs/impl/AsyncJobManagerImpl.java | 55 ++++++++++++++++++- .../vm/snapshot/VMSnapshotManagerImpl.java | 7 +++ 10 files changed, 111 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/snapshot/VMSnapshotService.java b/api/src/main/java/com/cloud/vm/snapshot/VMSnapshotService.java index 84a56aaedd3..754e463e710 100644 --- a/api/src/main/java/com/cloud/vm/snapshot/VMSnapshotService.java +++ b/api/src/main/java/com/cloud/vm/snapshot/VMSnapshotService.java @@ -19,6 +19,7 @@ package com.cloud.vm.snapshot; import java.util.List; +import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.api.command.user.vmsnapshot.ListVMSnapshotCmd; import com.cloud.exception.ConcurrentOperationException; @@ -53,4 +54,6 @@ public interface VMSnapshotService { * @param id vm id */ boolean deleteVMSnapshotsFromDB(Long vmId, boolean unmanage); + + void updateOperationFailed(VMSnapshot vmSnapshot) throws NoTransitionException; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java index 2a2db4af3e5..d318707a29a 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java @@ -21,6 +21,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage; import java.util.List; import com.cloud.storage.DataStoreRole; +import com.cloud.utils.fsm.NoTransitionException; public interface SnapshotDataFactory { SnapshotInfo getSnapshot(long snapshotId, DataStore store); @@ -42,4 +43,6 @@ public interface SnapshotDataFactory { List listSnapshotOnCache(long snapshotId); SnapshotInfo getReadySnapshotOnCache(long snapshotId); + + void updateOperationFailed(long snapshotId) throws NoTransitionException; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VMSnapshotStrategy.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VMSnapshotStrategy.java index 6372aa3bd94..2939342ec27 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VMSnapshotStrategy.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VMSnapshotStrategy.java @@ -18,6 +18,7 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.snapshot.VMSnapshot; public interface VMSnapshotStrategy { @@ -44,4 +45,6 @@ public interface VMSnapshotStrategy { * @return true if vm snapshot removed from DB, false if not. */ boolean deleteVMSnapshotFromDB(VMSnapshot vmSnapshot, boolean unmanage); + + void updateOperationFailed(VMSnapshot vmSnapshot) throws NoTransitionException; } diff --git a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java index a01d4ee5cae..997b413c099 100644 --- a/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java +++ b/engine/components-api/src/main/java/com/cloud/vm/snapshot/VMSnapshotManager.java @@ -54,5 +54,4 @@ public interface VMSnapshotManager extends VMSnapshotService, Manager { boolean hasActiveVMSnapshotTasks(Long vmId); RestoreVMSnapshotCommand createRestoreCommand(UserVmVO userVm, List vmSnapshotVOs); - } diff --git a/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java b/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java index 0143aaa1e73..bbdbfb5b520 100644 --- a/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/snapshot/dao/VMSnapshotDao.java @@ -38,6 +38,8 @@ public interface VMSnapshotDao extends GenericDao, StateDao< VMSnapshotVO findByName(Long vmId, String name); List listByAccountId(Long accountId); + List searchByVms(List vmIds); + List searchRemovedByVms(List vmIds, Long batchSize); } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index 4d8919ccc48..cd314b0be63 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -23,11 +23,16 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.storage.Snapshot; +import com.cloud.storage.Volume; +import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.commons.collections.CollectionUtils; @@ -73,7 +78,7 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { for (SnapshotDataStoreVO snapshotDataStoreVO : allSnapshotsFromVolumeAndDataStore) { DataStore store = storeMgr.getDataStore(snapshotDataStoreVO.getDataStoreId(), role); SnapshotVO snapshot = snapshotDao.findById(snapshotDataStoreVO.getSnapshotId()); - if (snapshot == null){ //snapshot may have been removed; + if (snapshot == null) { //snapshot may have been removed; continue; } SnapshotObject info = SnapshotObject.getSnapshotObject(snapshot, store); @@ -107,8 +112,6 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { return infos; } - - @Override public SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole role) { SnapshotVO snapshot = snapshotDao.findById(snapshotId); @@ -202,4 +205,17 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { return snapObjs; } + @Override + public void updateOperationFailed(long snapshotId) throws NoTransitionException { + List snapshotStoreRefs = snapshotStoreDao.findBySnapshotId(snapshotId); + for (SnapshotDataStoreVO snapshotStoreRef : snapshotStoreRefs) { + SnapshotInfo snapshotInfo = getSnapshot(snapshotStoreRef.getSnapshotId(), snapshotStoreRef.getDataStoreId(), snapshotStoreRef.getRole()); + if (snapshotInfo != null) { + VolumeInfo volumeInfo = snapshotInfo.getBaseVolume(); + volumeInfo.stateTransit(Volume.Event.OperationFailed); + ((SnapshotObject)snapshotInfo).processEvent(Snapshot.Event.OperationFailed); + snapshotInfo.processEvent(ObjectInDataStoreStateMachine.Event.OperationFailed); + } + } + } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java index 09f569e6f19..1801c877893 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/DefaultVMSnapshotStrategy.java @@ -481,4 +481,14 @@ public class DefaultVMSnapshotStrategy extends ManagerBase implements VMSnapshot } return StrategyPriority.DEFAULT; } + + @Override + public void updateOperationFailed(VMSnapshot vmSnapshot) throws NoTransitionException { + try { + vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.OperationFailed); + } catch (NoTransitionException e) { + logger.debug("Failed to change vm snapshot state with event OperationFailed"); + throw e; + } + } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java index 1ec6e20fc9e..8afbcab9b4b 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java @@ -479,6 +479,16 @@ public class ScaleIOVMSnapshotStrategy extends ManagerBase implements VMSnapshot return vmSnapshotDao.remove(vmSnapshot.getId()); } + @Override + public void updateOperationFailed(VMSnapshot vmSnapshot) throws NoTransitionException { + try { + vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.OperationFailed); + } catch (NoTransitionException e) { + logger.debug("Failed to change vm snapshot state with event OperationFailed"); + throw e; + } + } + private void publishUsageEvent(String type, VMSnapshot vmSnapshot, UserVm userVm, VolumeObjectTO volumeTo) { VolumeVO volume = volumeDao.findById(volumeTo.getId()); Long diskOfferingId = volume.getDiskOfferingId(); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 47bf27bd6c4..80140b0d950 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -35,6 +35,11 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.SnapshotVO; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotService; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.context.CallContext; @@ -153,11 +158,15 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, @Inject private SnapshotDao _snapshotDao; @Inject + private VMSnapshotDao _vmSnapshotDao; + @Inject private SnapshotService snapshotSrv; @Inject private SnapshotDataFactory snapshotFactory; @Inject private SnapshotDetailsDao _snapshotDetailsDao; + @Inject + private VMSnapshotService _vmSnapshotService; @Inject private VolumeDataFactory volFactory; @@ -1149,6 +1158,10 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return cleanupVirtualMachine(job.getInstanceId()); case Network: return cleanupNetwork(job.getInstanceId()); + case Snapshot: + return cleanupSnapshot(job.getInstanceId()); + case VmSnapshot: + return cleanupVmSnapshot(job.getInstanceId()); } } catch (Exception e) { logger.warn("Error while cleaning up resource: [" + job.getInstanceType().toString() + "] with Id: " + job.getInstanceId(), e); @@ -1187,7 +1200,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return true; } - private boolean cleanupNetwork(final long networkId) throws Exception { + private boolean cleanupNetwork(final long networkId) { NetworkVO networkVO = networkDao.findById(networkId); if (networkVO == null) { logger.warn("Network not found. Skip Cleanup. NetworkId: " + networkId); @@ -1206,6 +1219,46 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, return true; } + private boolean cleanupSnapshot(final long snapshotId) { + SnapshotVO snapshotVO = _snapshotDao.findById(snapshotId); + if (snapshotVO == null) { + logger.warn("Snapshot not found. Skip Cleanup. SnapshotId: " + snapshotId); + return true; + } + if (Snapshot.State.Allocated.equals(snapshotVO.getState())) { + _snapshotDao.remove(snapshotId); + } + if (Snapshot.State.Creating.equals(snapshotVO.getState())) { + try { + snapshotFactory.updateOperationFailed(snapshotId); + } catch (NoTransitionException e) { + snapshotVO.setState(Snapshot.State.Error); + _snapshotDao.update(snapshotVO.getId(), snapshotVO); + } + } + return true; + } + + private boolean cleanupVmSnapshot(final long vmSnapshotId) { + VMSnapshotVO vmSnapshotVO = _vmSnapshotDao.findById(vmSnapshotId); + if (vmSnapshotVO == null) { + logger.warn("VM Snapshot not found. Skip Cleanup. VMSnapshotId: " + vmSnapshotId); + return true; + } + if (VMSnapshot.State.Allocated.equals(vmSnapshotVO.getState())) { + _vmSnapshotDao.remove(vmSnapshotId); + } + if (VMSnapshot.State.Creating.equals(vmSnapshotVO.getState())) { + try { + _vmSnapshotService.updateOperationFailed(vmSnapshotVO); + } catch (NoTransitionException e) { + vmSnapshotVO.setState(VMSnapshot.State.Error); + _vmSnapshotDao.update(vmSnapshotVO.getId(), vmSnapshotVO); + } + } + return true; + } + private void cleanupFailedVolumesCreatedFromSnapshots(final long volumeId) { try { VolumeDetailVO volumeDetail = _volumeDetailsDao.findDetail(volumeId, VolumeService.SNAPSHOT_ID); diff --git a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java index 907182edc2a..26f7603240f 100644 --- a/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/snapshot/VMSnapshotManagerImpl.java @@ -28,6 +28,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.storage.snapshot.SnapshotManager; +import com.cloud.utils.fsm.NoTransitionException; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; @@ -1387,6 +1388,12 @@ public class VMSnapshotManagerImpl extends MutualExclusiveIdsManagerBase impleme return true; } + @Override + public void updateOperationFailed(VMSnapshot vmSnapshot) throws NoTransitionException { + VMSnapshotStrategy strategy = findVMSnapshotStrategy(vmSnapshot); + strategy.updateOperationFailed(vmSnapshot); + } + @Override public String getConfigComponentName() { return VMSnapshotManager.class.getSimpleName(); From 1a223fd2bac9ede56f07cb60421566472760a894 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Tue, 23 Sep 2025 03:48:18 -0300 Subject: [PATCH 016/145] server: Fix VM import DB sequence issue on import failure (#11659) * Fix VM import DB sequence issue on import failure * Remove ununsed imports * Refactor to avoid duplicating the next ID for VM sequence --- .../main/java/com/cloud/vm/UserVmService.java | 26 ++++++- .../java/com/cloud/vm/UserVmManagerImpl.java | 53 +++++++++------ .../vm/UnmanagedVMsManagerImpl.java | 68 ++++++------------- .../vm/UnmanagedVMsManagerImplTest.java | 3 - 4 files changed, 80 insertions(+), 70 deletions(-) diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 72b18b70e18..dc9e8c1f0d8 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -503,7 +503,31 @@ public interface UserVmService { void collectVmNetworkStatistics (UserVm userVm); - UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceName, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, + /** + * Import VM into CloudStack + * @param zone importing zone + * @param host importing host + * @param template template for the imported VM + * @param instanceNameInternal set to null to CloudStack to autogenerate from the next available VM ID on database + * @param displayName display name for the imported VM + * @param owner owner of the imported VM + * @param userData user data for the imported VM + * @param caller caller account + * @param isDisplayVm true to display the imported VM + * @param keyboard keyboard distribution for the imported VM + * @param accountId account ID + * @param userId user ID + * @param serviceOffering service offering for the imported VM + * @param sshPublicKey ssh key for the imported VM + * @param hostName the name for the imported VM + * @param hypervisorType hypervisor type for the imported VM + * @param customParameters details for the imported VM + * @param powerState power state of the imported VM + * @param networkNicMap network to nic mapping + * @return the imported VM + * @throws InsufficientCapacityException in case of errors + */ + UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceNameInternal, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKey, final String hostName, final HypervisorType hypervisorType, final Map customParameters, final VirtualMachine.PowerState powerState, final LinkedHashMap> networkNicMap) throws InsufficientCapacityException; diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 794d28c7adc..5b3284c2c1e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -8989,34 +8989,47 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } + private String getInternalName(long accountId, long vmId) { + String instanceSuffix = _configDao.getValue(Config.InstanceName.key()); + if (instanceSuffix == null) { + instanceSuffix = "DEFAULT"; + } + return VirtualMachineName.getVmName(vmId, accountId, instanceSuffix); + } + @Override - public UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceName, final String displayName, + public UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceNameInternal, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKeys, final String hostName, final HypervisorType hypervisorType, final Map customParameters, final VirtualMachine.PowerState powerState, final LinkedHashMap> networkNicMap) throws InsufficientCapacityException { - if (zone == null) { - throw new InvalidParameterValueException("Unable to import virtual machine with invalid zone"); - } - if (host == null && hypervisorType == HypervisorType.VMware) { - throw new InvalidParameterValueException("Unable to import virtual machine with invalid host"); - } + return Transaction.execute((TransactionCallbackWithException) status -> { + if (zone == null) { + throw new InvalidParameterValueException("Unable to import virtual machine with invalid zone"); + } + if (host == null && hypervisorType == HypervisorType.VMware) { + throw new InvalidParameterValueException("Unable to import virtual machine with invalid host"); + } - final long id = _vmDao.getNextInSequence(Long.class, "id"); + final long id = _vmDao.getNextInSequence(Long.class, "id"); + String instanceName = StringUtils.isBlank(instanceNameInternal) ? + getInternalName(owner.getAccountId(), id) : + instanceNameInternal; - if (hostName != null) { - // Check is hostName is RFC compliant - checkNameForRFCCompliance(hostName); - } + if (hostName != null) { + // Check is hostName is RFC compliant + checkNameForRFCCompliance(hostName); + } - final String uuidName = _uuidMgr.generateUuid(UserVm.class, null); - final Host lastHost = powerState != VirtualMachine.PowerState.PowerOn ? host : null; - final Boolean dynamicScalingEnabled = checkIfDynamicScalingCanBeEnabled(null, serviceOffering, template, zone.getId()); - return commitUserVm(true, zone, host, lastHost, template, hostName, displayName, owner, - null, null, userData, null, null, isDisplayVm, keyboard, - accountId, userId, serviceOffering, template.getFormat().equals(ImageFormat.ISO), sshPublicKeys, networkNicMap, - id, instanceName, uuidName, hypervisorType, customParameters, - null, null, null, powerState, dynamicScalingEnabled, null, serviceOffering.getDiskOfferingId(), null); + final String uuidName = _uuidMgr.generateUuid(UserVm.class, null); + final Host lastHost = powerState != VirtualMachine.PowerState.PowerOn ? host : null; + final Boolean dynamicScalingEnabled = checkIfDynamicScalingCanBeEnabled(null, serviceOffering, template, zone.getId()); + return commitUserVm(true, zone, host, lastHost, template, hostName, displayName, owner, + null, null, userData, null, null, isDisplayVm, keyboard, + accountId, userId, serviceOffering, template.getFormat().equals(ImageFormat.ISO), sshPublicKeys, networkNicMap, + id, instanceName, uuidName, hypervisorType, customParameters, + null, null, null, powerState, dynamicScalingEnabled, null, serviceOffering.getDiskOfferingId(), null); + }); } @Override diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 865884c2538..981f83936d2 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -133,7 +133,6 @@ import com.cloud.vm.UserVmVO; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; -import com.cloud.vm.VirtualMachineName; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfileImpl; import com.cloud.vm.VmDetailConstants; @@ -1111,13 +1110,13 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } } - private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedInstance, final String instanceName, final DataCenter zone, final Cluster cluster, final HostVO host, + private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedInstance, final String instanceNameInternal, final DataCenter zone, final Cluster cluster, final HostVO host, final VirtualMachineTemplate template, final String displayName, final String hostName, final Account caller, final Account owner, final Long userId, final ServiceOfferingVO serviceOffering, final Map dataDiskOfferingMap, final Map nicNetworkMap, final Map callerNicIpAddressMap, final Map details, final boolean migrateAllowed, final boolean forced, final boolean isImportUnmanagedFromSameHypervisor) { logger.debug(LogUtils.logGsonWithoutException("Trying to import VM [%s] with name [%s], in zone [%s], cluster [%s], and host [%s], using template [%s], service offering [%s], disks map [%s], NICs map [%s] and details [%s].", - unmanagedInstance, instanceName, zone, cluster, host, template, serviceOffering, dataDiskOfferingMap, nicNetworkMap, details)); + unmanagedInstance, displayName, zone, cluster, host, template, serviceOffering, dataDiskOfferingMap, nicNetworkMap, details)); UserVm userVm = null; ServiceOfferingVO validatedServiceOffering = null; try { @@ -1129,8 +1128,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } String internalCSName = unmanagedInstance.getInternalCSName(); - if (StringUtils.isEmpty(internalCSName)) { - internalCSName = instanceName; + if (StringUtils.isEmpty(instanceNameInternal)) { + internalCSName = instanceNameInternal; } Map allDetails = new HashMap<>(details); if (validatedServiceOffering.isDynamic()) { @@ -1142,18 +1141,18 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } if (!migrateAllowed && host != null && !hostSupportsServiceOfferingAndTemplate(host, validatedServiceOffering, template)) { - throw new InvalidParameterValueException(String.format("Service offering: %s or template: %s is not compatible with host: %s of unmanaged VM: %s", serviceOffering.getUuid(), template.getUuid(), host.getUuid(), instanceName)); + throw new InvalidParameterValueException(String.format("Service offering: %s or template: %s is not compatible with host: %s of unmanaged VM: %s", serviceOffering.getUuid(), template.getUuid(), host.getUuid(), displayName)); } // Check disks and supplied disk offerings List unmanagedInstanceDisks = unmanagedInstance.getDisks(); if (CollectionUtils.isEmpty(unmanagedInstanceDisks)) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("No attached disks found for the unmanaged VM: %s", instanceName)); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("No attached disks found for the unmanaged VM: %s", displayName)); } Pair> rootAndDataDisksPair = getRootAndDataDisks(unmanagedInstanceDisks, dataDiskOfferingMap); final UnmanagedInstanceTO.Disk rootDisk = rootAndDataDisksPair.first(); final List dataDisks = rootAndDataDisksPair.second(); if (rootDisk == null || StringUtils.isEmpty(rootDisk.getController())) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("VM import failed. Unable to retrieve root disk details for VM: %s ", instanceName)); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("VM import failed. Unable to retrieve root disk details for VM: %s ", displayName)); } if (cluster.getHypervisorType() == Hypervisor.HypervisorType.KVM) { Long rootDiskOfferingId = validatedServiceOffering.getDiskOfferingId(); @@ -1203,13 +1202,13 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { validatedServiceOffering, null, hostName, cluster.getHypervisorType(), allDetails, powerState, null); } catch (InsufficientCapacityException ice) { - String errorMsg = String.format("Failed to import VM [%s] due to [%s].", instanceName, ice.getMessage()); + String errorMsg = String.format("Failed to import VM [%s] due to [%s].", displayName, ice.getMessage()); logger.error(errorMsg, ice); throw new ServerApiException(ApiErrorCode.INSUFFICIENT_CAPACITY_ERROR, errorMsg); } if (userVm == null) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import vm name: %s", instanceName)); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import vm name: %s", displayName)); } List> diskProfileStoragePoolList = new ArrayList<>(); try { @@ -1239,9 +1238,9 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { deviceId++; } } catch (Exception e) { - logger.error(String.format("Failed to import volumes while importing vm: %s", instanceName), e); + logger.error(String.format("Failed to import volumes while importing vm: %s", displayName), e); cleanupFailedImportVM(userVm); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import volumes while importing vm: %s. %s", instanceName, StringUtils.defaultString(e.getMessage()))); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import volumes while importing vm: %s. %s", displayName, StringUtils.defaultString(e.getMessage()))); } try { int nicIndex = 0; @@ -1252,9 +1251,9 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { nicIndex++; } } catch (Exception e) { - logger.error(String.format("Failed to import NICs while importing vm: %s", instanceName), e); + logger.error(String.format("Failed to import NICs while importing vm: %s", displayName), e); cleanupFailedImportVM(userVm); - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import NICs while importing vm: %s. %s", instanceName, StringUtils.defaultString(e.getMessage()))); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to import NICs while importing vm: %s. %s", displayName, StringUtils.defaultString(e.getMessage()))); } if (migrateAllowed) { userVm = migrateImportedVM(host, template, validatedServiceOffering, userVm, owner, diskProfileStoragePoolList); @@ -1668,8 +1667,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { checkConversionSupportOnHost(convertHost, sourceVMName, true); } - String instanceName = getGeneratedInstanceName(owner); - checkNetworkingBeforeConvertingVmwareInstance(zone, owner, instanceName, hostName, sourceVMwareInstance, nicNetworkMap, nicIpAddressMap, forced); + checkNetworkingBeforeConvertingVmwareInstance(zone, owner, displayName, hostName, sourceVMwareInstance, nicNetworkMap, nicIpAddressMap, forced); UnmanagedInstanceTO convertedInstance; if (cmd.getForceMsToImportVmFiles() || !conversionSupportAnswer.isOvfExportSupported()) { // Uses MS for OVF export to temporary conversion location @@ -1690,14 +1688,14 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } sanitizeConvertedInstance(convertedInstance, sourceVMwareInstance); - UserVm userVm = importVirtualMachineInternal(convertedInstance, instanceName, zone, destinationCluster, null, + UserVm userVm = importVirtualMachineInternal(convertedInstance, null, zone, destinationCluster, null, template, displayName, hostName, caller, owner, userId, serviceOffering, dataDiskOfferingMap, nicNetworkMap, nicIpAddressMap, details, false, forced, false); long timeElapsedInSecs = (System.currentTimeMillis() - importStartTime) / 1000; logger.debug(String.format("VMware VM %s imported successfully to CloudStack instance %s (%s), Time taken: %d secs, OVF files imported from %s, Source VMware VM details - OS: %s, PowerState: %s, Disks: %s, NICs: %s", - sourceVMName, instanceName, displayName, timeElapsedInSecs, (ovfTemplateOnConvertLocation != null)? "MS" : "KVM Host", sourceVMwareInstance.getOperatingSystem(), sourceVMwareInstance.getPowerState(), sourceVMwareInstance.getDisks(), sourceVMwareInstance.getNics())); + sourceVMName, displayName, displayName, timeElapsedInSecs, (ovfTemplateOnConvertLocation != null)? "MS" : "KVM Host", sourceVMwareInstance.getOperatingSystem(), sourceVMwareInstance.getPowerState(), sourceVMwareInstance.getDisks(), sourceVMwareInstance.getNics())); return userVm; } catch (CloudRuntimeException e) { logger.error(String.format("Error importing VM: %s", e.getMessage()), e); @@ -1714,7 +1712,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } } - private void checkNetworkingBeforeConvertingVmwareInstance(DataCenter zone, Account owner, String instanceName, + private void checkNetworkingBeforeConvertingVmwareInstance(DataCenter zone, Account owner, String displayName, String hostName, UnmanagedInstanceTO sourceVMwareInstance, Map nicNetworkMap, Map nicIpAddressMap, @@ -1742,9 +1740,9 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } boolean autoImport = ipAddresses != null && ipAddresses.getIp4Address() != null && ipAddresses.getIp4Address().equalsIgnoreCase("auto"); checkUnmanagedNicAndNetworkMacAddressForImport(network, nic, forced); - checkUnmanagedNicAndNetworkForImport(instanceName, nic, network, zone, owner, autoImport, Hypervisor.HypervisorType.KVM); - checkUnmanagedNicAndNetworkHostnameForImport(instanceName, nic, network, hostName); - checkUnmanagedNicIpAndNetworkForImport(instanceName, nic, network, ipAddresses); + checkUnmanagedNicAndNetworkForImport(displayName, nic, network, zone, owner, autoImport, Hypervisor.HypervisorType.KVM); + checkUnmanagedNicAndNetworkHostnameForImport(displayName, nic, network, hostName); + checkUnmanagedNicIpAndNetworkForImport(displayName, nic, network, ipAddresses); } } @@ -1758,15 +1756,6 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } } - private String getGeneratedInstanceName(Account owner) { - long id = vmDao.getNextInSequence(Long.class, "id"); - String instanceSuffix = configurationDao.getValue(Config.InstanceName.key()); - if (instanceSuffix == null) { - instanceSuffix = "DEFAULT"; - } - return VirtualMachineName.getVmName(id, owner.getId(), instanceSuffix); - } - private void sanitizeConvertedInstance(UnmanagedInstanceTO convertedInstance, UnmanagedInstanceTO sourceVMwareInstance) { convertedInstance.setCpuCores(sourceVMwareInstance.getCpuCores()); convertedInstance.setCpuSpeed(sourceVMwareInstance.getCpuSpeed()); @@ -2512,10 +2501,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } VirtualMachine.PowerState powerState = VirtualMachine.PowerState.PowerOff; - String internalName = getInternalName(owner.getAccountId()); - try { - userVm = userVmManager.importVM(zone, null, template, internalName, displayName, owner, + userVm = userVmManager.importVM(zone, null, template, null, displayName, owner, null, caller, true, null, owner.getAccountId(), userId, serviceOffering, null, hostName, Hypervisor.HypervisorType.KVM, allDetails, powerState, null); @@ -2654,10 +2641,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { profiles.add(nicProfile); networkNicMap.put(network.getUuid(), profiles); - String internalName = getInternalName(owner.getAccountId()); - try { - userVm = userVmManager.importVM(zone, null, template, internalName, displayName, owner, + userVm = userVmManager.importVM(zone, null, template, null, displayName, owner, null, caller, true, null, owner.getAccountId(), userId, serviceOffering, null, hostName, Hypervisor.HypervisorType.KVM, allDetails, powerState, networkNicMap); @@ -2868,15 +2853,6 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { return getRemoteVmsAnswer.getUnmanagedInstances(); } - private String getInternalName(long accounId) { - String instanceSuffix = configurationDao.getValue(Config.InstanceName.key()); - if (instanceSuffix == null) { - instanceSuffix = "DEFAULT"; - } - long vmId = userVmDao.getNextInSequence(Long.class, "id"); - return VirtualMachineName.getVmName(vmId, accounId, instanceSuffix); - } - @Override public String getConfigComponentName() { return UnmanagedVMsManagerImpl.class.getSimpleName(); diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index 4ea6af7c6f4..4aa5df9d411 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -507,7 +507,6 @@ public class UnmanagedVMsManagerImplTest { VMTemplateVO template = Mockito.mock(VMTemplateVO.class); when(templateDao.findByName(anyString())).thenReturn(template); HostVO host = Mockito.mock(HostVO.class); - when(userVmDao.getNextInSequence(Long.class, "id")).thenReturn(1L); DeployDestination mockDest = Mockito.mock(DeployDestination.class); when(deploymentPlanningManager.planDeployment(any(), any(), any(), any())).thenReturn(mockDest); DiskProfile diskProfile = Mockito.mock(DiskProfile.class); @@ -593,7 +592,6 @@ public class UnmanagedVMsManagerImplTest { String tmplFileName = "5b8d689a-e61a-4ac3-9b76-e121ff90fbd3"; long newVmId = 2L; long networkId = 1L; - when(vmDao.getNextInSequence(Long.class, "id")).thenReturn(newVmId); ClusterVO cluster = mock(ClusterVO.class); when(cluster.getId()).thenReturn(clusterId); @@ -744,7 +742,6 @@ public class UnmanagedVMsManagerImplTest { when(hostDao.findById(anyLong())).thenReturn(host); NetworkOffering netOffering = Mockito.mock(NetworkOffering.class); when(entityMgr.findById(NetworkOffering.class, 0L)).thenReturn(netOffering); - when(userVmDao.getNextInSequence(Long.class, "id")).thenReturn(1L); DeployDestination mockDest = Mockito.mock(DeployDestination.class); when(deploymentPlanningManager.planDeployment(any(), any(), any(), any())).thenReturn(mockDest); DiskProfile diskProfile = Mockito.mock(DiskProfile.class); From 4884f52c9088cdf20b3638480412ec49eaad10b5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 23 Sep 2025 14:51:42 +0530 Subject: [PATCH 017/145] ui: allow provisioning backups during instance deploy (#11612) * ui: allow assigning backup offring during instance deploy Add backup offering selection to Deploy VM wizard and assign selected backup offering to the VM after successful deployment. This enables users to choose a backup offering during VM creation, and the VM is automatically associated with the selected offering post-deployment. Signed-off-by: Abhishek Kumar * changes for schedules Signed-off-by: Abhishek Kumar * fix Signed-off-by: Abhishek Kumar * fix Signed-off-by: Abhishek Kumar * Update ui/public/locales/pt_BR.json * Update ui/src/views/compute/wizard/DeployInstanceBackupSelection.vue Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * address review Signed-off-by: Abhishek Kumar * fix * allow only one schdeule per interval type Signed-off-by: Abhishek Kumar * show message same internaltype schedule * show backup step only when zone has offering --------- Signed-off-by: Abhishek Kumar Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- ui/public/locales/de_DE.json | 1 + ui/public/locales/el_GR.json | 1 + ui/public/locales/en.json | 9 +- ui/public/locales/es.json | 1 + ui/public/locales/ja_JP.json | 1 + ui/public/locales/ko_KR.json | 1 + ui/public/locales/pt_BR.json | 3 +- ui/public/locales/te.json | 1 + ui/public/locales/zh_CN.json | 1 + .../widgets/InfiniteScrollSelect.vue | 2 +- ui/src/config/section/storage.js | 2 +- ui/src/main.js | 4 +- ui/src/utils/plugins.js | 11 ++ ui/src/views/compute/BackupScheduleWizard.vue | 4 +- ui/src/views/compute/DeployVM.vue | 173 +++++++++++++++++- .../views/compute/backup/BackupSchedule.vue | 10 +- ui/src/views/compute/backup/FormSchedule.vue | 24 ++- .../wizard/DeployInstanceBackupSelection.vue | 154 ++++++++++++++++ 18 files changed, 384 insertions(+), 19 deletions(-) create mode 100644 ui/src/views/compute/wizard/DeployInstanceBackupSelection.vue diff --git a/ui/public/locales/de_DE.json b/ui/public/locales/de_DE.json index fd29ac3eb54..d99ba4157df 100644 --- a/ui/public/locales/de_DE.json +++ b/ui/public/locales/de_DE.json @@ -259,6 +259,7 @@ "label.available": "Verfügbar", "label.back": "Zurück", "label.backup": "Backup", +"label.backups": "Backup", "label.backup.attach.restore": "Backup-Volume wiederherstellen und anhängen", "label.backup.offering.assign": "VM zum Backup-Angebot zuordnen", "label.backup.offering.remove": "VM vom Backup-Angebot entfernen", diff --git a/ui/public/locales/el_GR.json b/ui/public/locales/el_GR.json index 6b939f52029..3c0cd6a1706 100644 --- a/ui/public/locales/el_GR.json +++ b/ui/public/locales/el_GR.json @@ -317,6 +317,7 @@ "label.availableprocessors": "Διαθέσιμοι πυρήνες επεξεργαστή", "label.back": "Πίσω", "label.backup": "Αντίγραφα ασφαλείας", +"label.backups": "Αντίγραφα ασφαλείας", "label.backup.attach.restore": "Επαναφορά και επισύναψη τόμου αντιγράφου ασφαλείας", "label.backup.offering.assign": "Αντιστοίχιση εικονικής μηχανής με προσφορά δημιουργίας αντιγράφων ασφαλείας", "label.backup.offering.remove": "Κατάργηση εικονικής μηχανής από την προσφορά δημιουργίας αντιγράφων ασφαλείας", diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 5136f241fb3..b740272bcad 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -252,6 +252,7 @@ "label.add.acl.rule": "Add ACL rule", "label.add.acl": "Add ACL", "label.add.affinity.group": "Add new Affinity Group", +"label.add.backup.schedule": "Add Backup Schedule", "label.add.baremetal.dhcp.device": "Add bare metal DHCP device", "label.add.bgp.peer": "Add BGP Peer", "label.add.bigswitchbcf.device": "Add BigSwitch BCF Controller", @@ -442,14 +443,17 @@ "label.availablevirtualmachinecount": "Available Instances", "label.back": "Back", "label.back.login": "Back to login", -"label.backup": "Backups", +"label.backup": "Backup", +"label.backups": "Backups", "label.backup.attach.restore": "Restore and attach backup volume", "label.backup.configure.schedule": "Configure Backup Schedule", "label.backup.offering.assign": "Assign Instance to backup offering", "label.backup.offering.remove": "Remove Instance from backup offering", "label.backup.offerings": "Backup Offerings", +"label.backup.offering.assign.failed": "Failed to assign Backup Offering", "label.backup.repository": "Backup Repository", "label.backup.restore": "Restore Instance backup", +"label.backup.schedule.create.failed": "Failed to create Backup Schedule", "label.backuplimit": "Backup Limits", "label.backup.storage": "Backup Storage", "label.backupstoragelimit": "Backup Storage Limits (GiB)", @@ -2194,6 +2198,7 @@ "label.select.all": "Select all", "label.select.columns": "Select columns", "label.select.a.zone": "Select a Zone", +"label.select.backup.offering": "Select Backup Offering", "label.select.deployment.infrastructure": "Select deployment infrastructure", "label.select.guest.os.type": "Please select the guest OS type", "label.select.network": "Select Network", @@ -3075,7 +3080,9 @@ "message.backup.attach.restore": "Please confirm that you want to restore and attach the volume from the backup?", "message.backup.create": "Are you sure you want to create an Instance backup?", "message.backup.offering.remove": "Are you sure you want to remove Instance from backup offering and delete the backup chain?", +"message.backup.provision.instance": "Select a backup offering to assign to the Instance. You can also add one or more backup schedules for different interval types to automate backups for this Instance. Assigning a backup offering and schedules helps protect your data by enabling automated and scheduled backups.", "message.backup.restore": "Please confirm that you want to restore the Instance backup?", +"message.backup.update.existing.schedule": "Updating existing backup schedule for the same interval type", "message.cancel.shutdown": "Please confirm that you would like to cancel the shutdown on this Management Server. It will resume accepting any new Async Jobs.", "message.cancel.maintenance": "Please confirm that you would like to cancel the maintenance on this Management Server. It will resume accepting any new Async Jobs.", "message.certificate.upload.processing": "Certificate upload in progress", diff --git a/ui/public/locales/es.json b/ui/public/locales/es.json index ab57c951531..b7a409adad1 100644 --- a/ui/public/locales/es.json +++ b/ui/public/locales/es.json @@ -222,6 +222,7 @@ "label.available": "Disponible", "label.back": "Volver", "label.backup": "Respaldos", +"label.backups": "Respaldos", "label.backup.attach.restore": "Restaurar y conectar un Volumen de Respaldo", "label.backup.offering.assign": "Asignar instancia a una oferta de respaldo", "label.backup.offering.remove": "remover instancia de una oferta de respaldo", diff --git a/ui/public/locales/ja_JP.json b/ui/public/locales/ja_JP.json index 3ab374a2a06..734a0ea851d 100644 --- a/ui/public/locales/ja_JP.json +++ b/ui/public/locales/ja_JP.json @@ -485,6 +485,7 @@ "label.available.public.ips": "使用できるパブリックIPアドレス", "label.back": "戻る", "label.backup": "バックアップ", + "label.backups": "バックアップ", "label.backup.attach.restore": "復元とバックアップボリュームをアタッチ", "label.backup.offering.assign": "VMをバックアップオファリングに割り当て", "label.backup.offering.remove": "VMバックアップオファリングから削除", diff --git a/ui/public/locales/ko_KR.json b/ui/public/locales/ko_KR.json index cc9683633be..1f20c95438d 100644 --- a/ui/public/locales/ko_KR.json +++ b/ui/public/locales/ko_KR.json @@ -269,6 +269,7 @@ "label.available": "\uc0ac\uc6a9 \uac00\ub2a5", "label.back": "\ub4a4\ub85c", "label.backup": "\ubc31\uc5c5", +"label.backups": "\ubc31\uc5c5", "label.backup.attach.restore": "\ubc31\uc5c5 \ubcfc\ub968 \ubcf5\uc6d0 \ubc0f \uc5f0\uacb0", "label.backup.offering.assign": "\uac00\uc0c1\uba38\uc2e0\uc5d0 \ubc31\uc5c5 \uc624\ud37c\ub9c1 \ud560\ub2f9", "label.backup.offering.remove": "\uac00\uc0c1\uba38\uc2e0\uc5d0 \ubc31\uc5c5 \uc624\ud37c\ub9c1 \uc81c\uac70", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 09b0242ef1a..711dd36deff 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -292,7 +292,8 @@ "label.availability": "Disponibilidade", "label.available": "Dispon\u00edvel", "label.back": "Voltar", -"label.backup": "Backups", +"label.backup": "Backup", +"label.backups": "Backups", "label.backup.attach.restore": "Restaurar e anexar volume de backup", "label.backup.offering.assign": "Atribuir VM a oferta de backup", "label.backup.offering.remove": "Remover VM de oferta de backup", diff --git a/ui/public/locales/te.json b/ui/public/locales/te.json index 957aed4ee88..5f89bbf7ed7 100644 --- a/ui/public/locales/te.json +++ b/ui/public/locales/te.json @@ -419,6 +419,7 @@ "label.back": "వెనుకకు", "label.back.login": "తిరిగి లాగిన్‌కి", "label.backup": "బ్యాకప్‌లు", + "label.backups": "బ్యాకప్‌లు", "label.backup.attach.restore": "బ్యాకప్ వాల్యూమ్‌ను పునరుద్ధరించండి మరియు అటాచ్ చేయండి", "label.backup.configure.schedule": "బ్యాకప్ షెడ్యూల్‌ను కాన్ఫిగర్ చేయండి", "label.backup.offering.assign": "బ్యాకప్ సమర్పణకు ఉదాహరణను కేటాయించండి", diff --git a/ui/public/locales/zh_CN.json b/ui/public/locales/zh_CN.json index d09f960edf5..02149360841 100644 --- a/ui/public/locales/zh_CN.json +++ b/ui/public/locales/zh_CN.json @@ -561,6 +561,7 @@ "label.back": "\u540E\u9000", "label.backup": "\u5907\u4EFD", + "label.backups": "\u5907\u4EFD", "label.backup.attach.restore": "\u6062\u590D\u5E76\u8FDE\u63A5\u5907\u4EFD\u5377", "label.backup.offering.assign": "\u5C06\u865A\u62DF\u673A\u5206\u914D\u7ED9\u5907\u4EFD\u4EA7\u54C1", "label.backup.offering.remove": "\u4ECE\u5907\u4EFD\u4EA7\u54C1\u4E2D\u5220\u9664\u865A\u62DF\u673A", diff --git a/ui/src/components/widgets/InfiniteScrollSelect.vue b/ui/src/components/widgets/InfiniteScrollSelect.vue index c11f9686446..608eeebf133 100644 --- a/ui/src/components/widgets/InfiniteScrollSelect.vue +++ b/ui/src/components/widgets/InfiniteScrollSelect.vue @@ -77,7 +77,7 @@ - + diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 4308832bcf2..df0176e5a2d 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -415,7 +415,7 @@ export default { }, { name: 'backup', - title: 'label.backup', + title: 'label.backups', icon: 'cloud-upload-outlined', permission: ['listBackups'], params: { listvmdetails: 'true' }, diff --git a/ui/src/main.js b/ui/src/main.js index f117fb57810..7441f801086 100644 --- a/ui/src/main.js +++ b/ui/src/main.js @@ -40,7 +40,8 @@ import { dialogUtilPlugin, cpuArchitectureUtilPlugin, imagesUtilPlugin, - extensionsUtilPlugin + extensionsUtilPlugin, + backupUtilPlugin } from './utils/plugins' import { VueAxios } from './utils/request' import directives from './utils/directives' @@ -63,6 +64,7 @@ vueApp.use(dialogUtilPlugin) vueApp.use(cpuArchitectureUtilPlugin) vueApp.use(imagesUtilPlugin) vueApp.use(extensionsUtilPlugin) +vueApp.use(backupUtilPlugin) vueApp.use(extensions) vueApp.use(directives) diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index 61456d98b12..648bc3ae081 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -597,3 +597,14 @@ export const extensionsUtilPlugin = { } } } + +export const backupUtilPlugin = { + install (app) { + app.config.globalProperties.$isBackupProviderSupportsQuiesceVm = function (provider) { + if (!provider && typeof provider !== 'string') { + return false + } + return ['nas'].includes(provider.toLowerCase()) + } + } +} diff --git a/ui/src/views/compute/BackupScheduleWizard.vue b/ui/src/views/compute/BackupScheduleWizard.vue index a37893d2bc1..650f4915ece 100644 --- a/ui/src/views/compute/BackupScheduleWizard.vue +++ b/ui/src/views/compute/BackupScheduleWizard.vue @@ -21,13 +21,11 @@ + :resource="resource"/> diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index 7872a405a69..854d082075f 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -540,6 +540,25 @@ + + + @@ -901,7 +920,7 @@ From a749206eb8e992f8cbc0d82495c7bbc713ff470e Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 24 Sep 2025 11:52:49 +0530 Subject: [PATCH 018/145] storage: Mount disabled pools by default when host is booted (#11666) --- .../src/main/java/com/cloud/storage/StorageManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 7b31ec6a81b..529e506e8a0 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -182,7 +182,7 @@ public interface StorageManager extends StorageService { ConfigKey MountDisabledStoragePool = new ConfigKey<>(Boolean.class, "mount.disabled.storage.pool", "Storage", - "false", + Boolean.TRUE.toString(), "Mount all zone-wide or cluster-wide disabled storage pools after node reboot", true, ConfigKey.Scope.Cluster, From a18b5514e67a248f57f6e212822a4a44e0b6195c Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 24 Sep 2025 12:04:18 +0530 Subject: [PATCH 019/145] kvm: honor templateId passed in importVM API (#11640) --- .../apache/cloudstack/vm/UnmanagedVMsManagerImpl.java | 11 ++--------- .../cloudstack/vm/UnmanagedVMsManagerImplTest.java | 4 ---- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 981f83936d2..30cf4ad76a7 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -1493,7 +1493,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { if (templateId == null) { template = templateDao.findByName(VM_IMPORT_DEFAULT_TEMPLATE_NAME); if (template == null) { - template = createDefaultDummyVmImportTemplate(false); + template = createDefaultDummyVmImportTemplate(Hypervisor.HypervisorType.KVM == hypervisorType); if (template == null) { throw new InvalidParameterValueException(String.format("Default VM import template with unique name: %s for hypervisor: %s cannot be created. Please use templateid parameter for import", VM_IMPORT_DEFAULT_TEMPLATE_NAME, hypervisorType.toString())); } @@ -2331,14 +2331,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { if (CollectionUtils.isNotEmpty(userVOs)) { userId = userVOs.get(0).getId(); } - VMTemplateVO template = templateDao.findByName(KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME); - if (template == null) { - template = createDefaultDummyVmImportTemplate(true); - if (template == null) { - throw new InvalidParameterValueException("Error while creating default Import Vm Template"); - } - } - + VMTemplateVO template = getTemplateForImportInstance(cmd.getTemplateId(), Hypervisor.HypervisorType.KVM); final Long serviceOfferingId = cmd.getServiceOfferingId(); if (serviceOfferingId == null) { throw new InvalidParameterValueException(String.format("Service offering ID cannot be null")); diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index 4aa5df9d411..09f62f7a049 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -504,8 +504,6 @@ public class UnmanagedVMsManagerImplTest { when(cmd.getPassword()).thenReturn("pass"); when(cmd.getImportSource()).thenReturn("external"); when(cmd.getDomainId()).thenReturn(null); - VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - when(templateDao.findByName(anyString())).thenReturn(template); HostVO host = Mockito.mock(HostVO.class); DeployDestination mockDest = Mockito.mock(DeployDestination.class); when(deploymentPlanningManager.planDeployment(any(), any(), any(), any())).thenReturn(mockDest); @@ -736,8 +734,6 @@ public class UnmanagedVMsManagerImplTest { when(cmd.getImportSource()).thenReturn(source); when(cmd.getDiskPath()).thenReturn("/var/lib/libvirt/images/test.qcow2"); when(cmd.getDomainId()).thenReturn(null); - VMTemplateVO template = Mockito.mock(VMTemplateVO.class); - when(templateDao.findByName(anyString())).thenReturn(template); HostVO host = Mockito.mock(HostVO.class); when(hostDao.findById(anyLong())).thenReturn(host); NetworkOffering netOffering = Mockito.mock(NetworkOffering.class); From 98b9af29040533089a7f8153c74f95f9c324d66b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 24 Sep 2025 12:51:40 +0530 Subject: [PATCH 020/145] server: set VirtualMachineTO arch from template if present (#11530) * server: set VirtualMachineTO arch from template if present Fixes #11529 Signed-off-by: Abhishek Kumar * refactor Signed-off-by: Abhishek Kumar --------- Signed-off-by: Abhishek Kumar --- .../com/cloud/hypervisor/HypervisorGuruBase.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java index c510502f5f9..4f14fe20dac 100644 --- a/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java +++ b/server/src/main/java/com/cloud/hypervisor/HypervisorGuruBase.java @@ -23,6 +23,7 @@ import java.util.UUID; import javax.inject.Inject; +import com.cloud.cpu.CPU; import com.cloud.dc.DataCenter; import com.cloud.dc.dao.DataCenterDao; import com.cloud.domain.Domain; @@ -307,10 +308,15 @@ public abstract class HypervisorGuruBase extends AdapterBase implements Hypervis to.setNics(nics); to.setDisks(vmProfile.getDisks().toArray(new DiskTO[vmProfile.getDisks().size()])); - if (vmProfile.getTemplate().getBits() == 32) { - to.setArch("i686"); + CPU.CPUArch templateArch = vmProfile.getTemplate().getArch(); + if (templateArch != null) { + to.setArch(templateArch.getType()); } else { - to.setArch("x86_64"); + if (vmProfile.getTemplate().getBits() == 32) { + to.setArch(CPU.CPUArch.x86.getType()); + } else { + to.setArch(CPU.CPUArch.amd64.getType()); + } } Map detailsInVm = _userVmDetailsDao.listDetailsKeyPairs(vm.getId()); From 96992d3d640967b62ca5e323890ca4052d05b072 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Wed, 24 Sep 2025 13:58:24 +0530 Subject: [PATCH 021/145] server: Fix vpclimit count for listAcccount API response (#11686) --- .../org/apache/cloudstack/api/response/AccountResponse.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java index 6fc098295f6..d761d7c0394 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AccountResponse.java @@ -464,7 +464,7 @@ public class AccountResponse extends BaseResponse implements ResourceLimitAndCou @Override public void setVpcLimit(String vpcLimit) { - this.vpcLimit = networkLimit; + this.vpcLimit = vpcLimit; } @Override From c24d2b88f6bff2a13b21cad8fc25ebf367c726be Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 24 Sep 2025 11:30:04 +0200 Subject: [PATCH 022/145] LDAP: honour nested groups for MSAD (#11696) --- .../org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java | 8 ++++++-- .../org/apache/cloudstack/ldap/LdapConfiguration.java | 5 ++++- .../apache/cloudstack/ldap/OpenLdapUserManagerImpl.java | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java index 552d5969a9e..e96606dca2f 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/ADLdapUserManagerImpl.java @@ -93,10 +93,14 @@ public class ADLdapUserManagerImpl extends OpenLdapUserManagerImpl implements Ld } protected String getMemberOfAttribute(final Long domainId) { + String rc; if(_ldapConfiguration.isNestedGroupsEnabled(domainId)) { - return MICROSOFT_AD_NESTED_MEMBERS_FILTER; + rc = MICROSOFT_AD_NESTED_MEMBERS_FILTER; } else { - return MICROSOFT_AD_MEMBERS_FILTER; + rc = MICROSOFT_AD_MEMBERS_FILTER; } + logger.trace("using memberOf filter = {} for domain with id {}", rc, domainId); + + return rc; } } diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java index 6a62ad8d99d..87ff2d0a2ac 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapConfiguration.java @@ -27,9 +27,12 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import com.cloud.utils.Pair; import org.apache.cloudstack.ldap.dao.LdapConfigurationDao; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class LdapConfiguration implements Configurable{ private final static String factory = "com.sun.jndi.ldap.LdapCtxFactory"; + protected Logger logger = LogManager.getLogger(getClass()); private static final ConfigKey ldapReadTimeout = new ConfigKey( Long.class, @@ -325,7 +328,7 @@ public class LdapConfiguration implements Configurable{ try { provider = LdapUserManager.Provider.valueOf(ldapProvider.valueIn(domainId).toUpperCase()); } catch (IllegalArgumentException ex) { - //openldap is the default + logger.warn("no LDAP provider found for domain {}, using openldap as default", domainId); provider = LdapUserManager.Provider.OPENLDAP; } return provider; diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java index 4c125af2ea6..d0b6bc4bd34 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/OpenLdapUserManagerImpl.java @@ -63,7 +63,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { final String firstname = LdapUtils.getAttributeValue(attributes, _ldapConfiguration.getFirstnameAttribute(domainId)); final String lastname = LdapUtils.getAttributeValue(attributes, _ldapConfiguration.getLastnameAttribute(domainId)); final String principal = result.getNameInNamespace(); - final List memberships = LdapUtils.getAttributeValues(attributes, _ldapConfiguration.getUserMemberOfAttribute(domainId)); + final List memberships = LdapUtils.getAttributeValues(attributes, getMemberOfAttribute(domainId)); String domain = principal.replace("cn=" + LdapUtils.getAttributeValue(attributes, _ldapConfiguration.getCommonNameAttribute()) + ",", ""); domain = domain.replace("," + _ldapConfiguration.getBaseDn(domainId), ""); @@ -87,7 +87,7 @@ public class OpenLdapUserManagerImpl implements LdapUserManager { usernameFilter.append((username == null ? "*" : LdapUtils.escapeLDAPSearchFilter(username))); usernameFilter.append(")"); - String memberOfAttribute = _ldapConfiguration.getUserMemberOfAttribute(domainId); + String memberOfAttribute = getMemberOfAttribute(domainId); StringBuilder ldapGroupsFilter = new StringBuilder(); // this should get the trustmaps for this domain List ldapGroups = getMappedLdapGroups(domainId); From 36cfd76ce199ce159f496b27f9ed080488b96427 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 24 Sep 2025 13:53:27 +0200 Subject: [PATCH 023/145] KVM: fix delete vm snapshot if it does not exist with a Stopped vm (#11687) * KVM: fix delete vm snapshot if it does not exist with a Stopped vm * update 11687 --- .../wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java index 58a74d6e0f6..9f1b264ff71 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteVMSnapshotCommandWrapper.java @@ -40,6 +40,7 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Volume; +import com.cloud.utils.StringUtils; import com.cloud.utils.script.Script; @ResourceWrapper(handles = DeleteVMSnapshotCommand.class) @@ -104,7 +105,7 @@ public final class LibvirtDeleteVMSnapshotCommandWrapper extends CommandWrapper< commands.add(new String[]{Script.getExecutableAbsolutePath("awk"), "-F", " ", "{print $2}"}); commands.add(new String[]{Script.getExecutableAbsolutePath("grep"), "^" + sanitizeBashCommandArgument(cmd.getTarget().getSnapshotName()) + "$"}); String qemu_img_snapshot = Script.executePipedCommands(commands, 0).second(); - if (qemu_img_snapshot == null) { + if (StringUtils.isEmpty(qemu_img_snapshot)) { logger.info("Cannot find snapshot " + cmd.getTarget().getSnapshotName() + " in file " + rootDisk.getPath() + ", return true"); return new DeleteVMSnapshotAnswer(cmd, cmd.getVolumeTOs()); } From b0c77190066fd74336ea831ccf9e5427e793ba7f Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 24 Sep 2025 21:12:28 +0530 Subject: [PATCH 024/145] ui: do not show admin only options to users while registering template (#11702) --- ui/src/views/image/RegisterOrUploadTemplate.vue | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index 4cb6aa10df8..188f8c45ae9 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -685,6 +685,9 @@ export default { }) }, fetchCustomHypervisorName () { + if (!('listConfigurations' in store.getters.apis)) { + return + } const params = { name: 'hypervisor.custom.display.name' } @@ -702,6 +705,9 @@ export default { }) }, fetchExtensionsList () { + if (!this.isAdminRole) { + return + } this.loading = true getAPI('listExtensions', { }).then(response => { @@ -758,6 +764,9 @@ export default { name: 'Simulator' }) } + if (!this.isAdminRole) { + listhyperVisors = listhyperVisors.filter(hv => hv.name !== 'External') + } this.hyperVisor.opts = listhyperVisors }).finally(() => { this.hyperVisor.loading = false From 23c9e830473ef9a45a28439ea7ec18b1f05205af Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Thu, 25 Sep 2025 13:28:29 +0530 Subject: [PATCH 025/145] Create Instance from backup on another Zone (DRaaS use case) (#11560) * draas initial changes * Added option to enable disaster recovery on a backup respository. Added UpdateBackupRepositoryCmd api. * Added timeout for mount operation in backup restore configurable via global setting * Addressed review comments * fix for simulator test failures * Added UT for coverage * Fix create instance from backup ui for other providers * Added events to add/update backup repository * Fix race in fetchZones * One more fix in fetchZones in DeployVMFromBackup.vue * Fix zone selection in createNetwork via Create Instance from backup form. * Allow template/iso selection in create instance from backup ui * rename draasenabled to crosszoneinstancecreation * Added Cross-zone instance creation in test_backup_recovery_nas.py * Added UT in BackupManagerTest and UserVmManagerImplTest * Integration test added for Cross-zone instance creation in test_backup_recovery_nas.py --- .../main/java/com/cloud/event/EventTypes.java | 9 + .../com/cloud/vm/VirtualMachineProfile.java | 1 + .../apache/cloudstack/api/ApiConstants.java | 1 + .../repository/AddBackupRepositoryCmd.java | 8 +- .../repository/UpdateBackupRepositoryCmd.java | 116 ++++ .../response/BackupRepositoryResponse.java | 12 + .../cloudstack/backup/BackupManager.java | 2 + .../cloudstack/backup/BackupProvider.java | 4 +- .../cloudstack/backup/BackupRepository.java | 3 + .../backup/BackupRepositoryService.java | 2 + .../backup/RestoreBackupCommand.java | 9 + .../cloud/vm/VirtualMachineManagerImpl.java | 15 + .../cloudstack/backup/BackupRepositoryVO.java | 21 +- .../META-INF/db/schema-42100to42200.sql | 3 + .../backup/DummyBackupProvider.java | 9 +- .../cloudstack/backup/NASBackupProvider.java | 73 ++- .../backup/NASBackupProviderTest.java | 138 ++++- .../backup/NetworkerBackupProvider.java | 9 +- .../backup/VeeamBackupProvider.java | 9 +- .../LibvirtRestoreBackupCommandWrapper.java | 103 ++-- ...ibvirtRestoreBackupCommandWrapperTest.java | 499 ++++++++++++++++++ .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../java/com/cloud/vm/UserVmManagerImpl.java | 49 +- .../cloudstack/backup/BackupManagerImpl.java | 50 +- .../backup/BackupRepositoryServiceImpl.java | 53 +- .../com/cloud/vm/UserVmManagerImplTest.java | 126 ++++- .../cloudstack/backup/BackupManagerTest.java | 274 +++++++++- .../BackupRepositoryServiceImplTest.java | 243 +++++++++ .../smoke/test_backup_recovery_nas.py | 124 +++-- tools/marvin/marvin/lib/base.py | 14 +- ui/public/locales/en.json | 4 + ui/src/components/view/DeployVMFromBackup.vue | 111 +++- ui/src/config/section/config.js | 13 +- .../compute/wizard/TemplateIsoRadioGroup.vue | 10 +- .../network/CreateIsolatedNetworkForm.vue | 2 +- ui/src/views/network/CreateL2NetworkForm.vue | 2 +- ui/src/views/network/CreateNetwork.vue | 2 +- .../views/network/CreateSharedNetworkForm.vue | 2 +- ui/src/views/storage/CreateVMFromBackup.vue | 41 +- 39 files changed, 1961 insertions(+), 206 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java create mode 100644 plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java create mode 100644 server/src/test/java/org/apache/cloudstack/backup/BackupRepositoryServiceImplTest.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index be21f13267b..38e601c790a 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.backup.BackupRepositoryService; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet; import org.apache.cloudstack.extension.Extension; @@ -852,6 +853,10 @@ public class EventTypes { // Custom Action public static final String EVENT_CUSTOM_ACTION = "CUSTOM.ACTION"; + // Backup Repository + public static final String EVENT_BACKUP_REPOSITORY_ADD = "BACKUP.REPOSITORY.ADD"; + public static final String EVENT_BACKUP_REPOSITORY_UPDATE = "BACKUP.REPOSITORY.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1385,6 +1390,10 @@ public class EventTypes { entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_UPDATE, ExtensionCustomAction.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_DELETE, ExtensionCustomAction.class); + + // Backup Repository + entityEventDetails.put(EVENT_BACKUP_REPOSITORY_ADD, BackupRepositoryService.class); + entityEventDetails.put(EVENT_BACKUP_REPOSITORY_UPDATE, BackupRepositoryService.class); } public static boolean isNetworkEvent(String eventType) { diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index c67ee4eabc2..5c78d6bedd6 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -78,6 +78,7 @@ public interface VirtualMachineProfile { public static final Param BootIntoSetup = new Param("enterHardwareSetup"); public static final Param PreserveNics = new Param("PreserveNics"); public static final Param ConsiderLastHost = new Param("ConsiderLastHost"); + public static final Param ReturnAfterVolumePrepare = new Param("ReturnAfterVolumePrepare"); private String name; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 489d737b5bb..6c84e54b2d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -139,6 +139,7 @@ public class ApiConstants { public static final String CPU_SPEED = "cpuspeed"; public static final String CPU_LOAD_AVERAGE = "cpuloadaverage"; public static final String CREATED = "created"; + public static final String CROSS_ZONE_INSTANCE_CREATION = "crosszoneinstancecreation"; public static final String CTX_ACCOUNT_ID = "ctxaccountid"; public static final String CTX_DETAILS = "ctxDetails"; public static final String CTX_USER_ID = "ctxuserid"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java index 5d0c838bc37..64998a74954 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/AddBackupRepositoryCmd.java @@ -63,12 +63,14 @@ public class AddBackupRepositoryCmd extends BaseCmd { type = CommandType.UUID, entityType = ZoneResponse.class, required = true, - description = "ID of the zone where the backup repository is to be added") + description = "ID of the zone where the backup repository is to be added for taking backups") private Long zoneId; @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, description = "capacity of this backup repository") private Long capacityBytes; + @Parameter(name = ApiConstants.CROSS_ZONE_INSTANCE_CREATION, type = CommandType.BOOLEAN, description = "backups on this repository can be used to create Instances on all Zones", since = "4.22.0") + private Boolean crossZoneInstanceCreation; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -109,6 +111,10 @@ public class AddBackupRepositoryCmd extends BaseCmd { return capacityBytes; } + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java new file mode 100644 index 00000000000..5ffd79e497e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/repository/UpdateBackupRepositoryCmd.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.backup.repository; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupRepositoryResponse; +import org.apache.cloudstack.backup.BackupRepository; +import org.apache.cloudstack.backup.BackupRepositoryService; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; + +@APICommand(name = "updateBackupRepository", + description = "Update a backup repository", + responseObject = BackupRepositoryResponse.class, since = "4.22.0", + authorized = {RoleType.Admin}) +public class UpdateBackupRepositoryCmd extends BaseCmd { + + @Inject + private BackupRepositoryService backupRepositoryService; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, entityType = BackupRepositoryResponse.class, description = "ID of the backup repository") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "name of the backup repository") + private String name; + + @Parameter(name = ApiConstants.ADDRESS, type = CommandType.STRING, description = "address of the backup repository") + private String address; + + @Parameter(name = ApiConstants.MOUNT_OPTIONS, type = CommandType.STRING, description = "shared storage mount options") + private String mountOptions; + + @Parameter(name = ApiConstants.CROSS_ZONE_INSTANCE_CREATION, type = CommandType.BOOLEAN, description = "backups in this repository can be used to create Instances on all Zones") + private Boolean crossZoneInstanceCreation; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public BackupRepositoryService getBackupRepositoryService() { + return backupRepositoryService; + } + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getAddress() { + return address; + } + + public String getMountOptions() { + return mountOptions == null ? "" : mountOptions; + } + + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + + ///////////////////////////////////////////////////// + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + try { + BackupRepository result = backupRepositoryService.updateBackupRepository(this); + if (result != null) { + BackupRepositoryResponse response = _responseGenerator.createBackupRepositoryResponse(result); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the backup repository"); + } + } catch (Exception ex4) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex4.getMessage()); + } + + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java index 0db51f04034..327bbae0051 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java @@ -61,6 +61,10 @@ public class BackupRepositoryResponse extends BaseResponse { @Param(description = "capacity of the backup repository") private Long capacityBytes; + @SerializedName(ApiConstants.CROSS_ZONE_INSTANCE_CREATION) + @Param(description = "the backups in this repository can be used to create Instances on all Zones") + private Boolean crossZoneInstanceCreation; + @SerializedName("created") @Param(description = "the date and time the backup repository was added") private Date created; @@ -132,6 +136,14 @@ public class BackupRepositoryResponse extends BaseResponse { this.capacityBytes = capacityBytes; } + public Boolean getCrossZoneInstanceCreation() { + return crossZoneInstanceCreation; + } + + public void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation) { + this.crossZoneInstanceCreation = crossZoneInstanceCreation; + } + public Date getCreated() { return created; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index c4b92fc9e05..37d21613c3d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -205,6 +205,8 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer Boolean canCreateInstanceFromBackup(Long backupId); + Boolean canCreateInstanceFromBackupAcrossZones(Long backupId); + /** * Restore a backup to a new Instance */ diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 1eb36f89556..32a714370df 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -23,6 +23,8 @@ import com.cloud.vm.VirtualMachine; public interface BackupProvider { + Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering); + /** * Returns the unique name of the provider * @return returns provider name @@ -85,7 +87,7 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); - boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); + Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); /** * Restore VM from backup diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java index be539a0eb04..886d13c13f9 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepository.java @@ -28,9 +28,12 @@ public interface BackupRepository extends InternalIdentity, Identity { String getType(); String getAddress(); String getMountOptions(); + void setMountOptions(String mountOptions); void setUsedBytes(Long usedBytes); Long getCapacityBytes(); Long getUsedBytes(); void setCapacityBytes(Long capacityBytes); + Boolean crossZoneInstanceCreationEnabled(); + void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation); Date getCreated(); } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java index ae71053e400..875fc3b3d90 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupRepositoryService.java @@ -23,11 +23,13 @@ import com.cloud.utils.Pair; import org.apache.cloudstack.api.command.user.backup.repository.AddBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.DeleteBackupRepositoryCmd; import org.apache.cloudstack.api.command.user.backup.repository.ListBackupRepositoriesCmd; +import org.apache.cloudstack.api.command.user.backup.repository.UpdateBackupRepositoryCmd; import java.util.List; public interface BackupRepositoryService { BackupRepository addBackupRepository(AddBackupRepositoryCmd cmd); + BackupRepository updateBackupRepository(UpdateBackupRepositoryCmd cmd); boolean deleteBackupRepository(DeleteBackupRepositoryCmd cmd); Pair, Integer> listBackupRepositories(ListBackupRepositoriesCmd cmd); diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f447fbe3d00..453b236df6b 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -36,6 +36,7 @@ public class RestoreBackupCommand extends Command { private Boolean vmExists; private String restoreVolumeUUID; private VirtualMachine.State vmState; + private Integer mountTimeout; protected RestoreBackupCommand() { super(); @@ -136,4 +137,12 @@ public class RestoreBackupCommand extends Command { public void setBackupVolumesUUIDs(List backupVolumesUUIDs) { this.backupVolumesUUIDs = backupVolumesUUIDs; } + + public Integer getMountTimeout() { + return this.mountTimeout; + } + + public void setMountTimeout(Integer mountTimeout) { + this.mountTimeout = mountTimeout; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 3a6e1b62277..b5597280364 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1482,6 +1482,21 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac volumeMgr.prepare(vmProfile, dest); } + if (params != null) { + Boolean returnAfterVolumePrepare = (Boolean) params.get(VirtualMachineProfile.Param.ReturnAfterVolumePrepare); + if (Boolean.TRUE.equals(returnAfterVolumePrepare)) { + logger.info("Returning from VM start command execution for VM {} as requested. Volumes are prepared and ready.", vm.getUuid()); + + if (!changeState(vm, Event.AgentReportStopped, destHostId, work, Step.Done)) { + logger.error("Unable to transition to a new state. VM uuid: {}, VM oldstate: {}, Event: {}", vm, vm.getState(), Event.AgentReportStopped); + throw new ConcurrentOperationException(String.format("Failed to deploy VM %s", vm)); + } + + logger.debug("Volume preparation completed for VM {} (VM state set to Stopped)", vm); + return; + } + } + if (!reuseVolume) { reuseVolume = true; } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java index 98efa94ceca..1764496a6c0 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupRepositoryVO.java @@ -67,6 +67,9 @@ public class BackupRepositoryVO implements BackupRepository { @Column(name = "capacity_bytes", nullable = true) private Long capacityBytes; + @Column(name = "cross_zone_instance_creation") + private Boolean crossZoneInstanceCreation; + @Column(name = "created") @Temporal(value = TemporalType.TIMESTAMP) private Date created; @@ -79,7 +82,7 @@ public class BackupRepositoryVO implements BackupRepository { this.uuid = UUID.randomUUID().toString(); } - public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes) { + public BackupRepositoryVO(final long zoneId, final String provider, final String name, final String type, final String address, final String mountOptions, final Long capacityBytes, final Boolean crossZoneInstanceCreation) { this(); this.zoneId = zoneId; this.provider = provider; @@ -88,6 +91,7 @@ public class BackupRepositoryVO implements BackupRepository { this.address = address; this.mountOptions = mountOptions; this.capacityBytes = capacityBytes; + this.crossZoneInstanceCreation = crossZoneInstanceCreation; this.created = new Date(); } @@ -139,6 +143,11 @@ public class BackupRepositoryVO implements BackupRepository { return mountOptions; } + @Override + public void setMountOptions(String mountOptions) { + this.mountOptions = mountOptions; + } + @Override public Long getUsedBytes() { return usedBytes; @@ -154,6 +163,16 @@ public class BackupRepositoryVO implements BackupRepository { return capacityBytes; } + @Override + public Boolean crossZoneInstanceCreationEnabled() { + return crossZoneInstanceCreation; + } + + @Override + public void setCrossZoneInstanceCreation(Boolean crossZoneInstanceCreation) { + this.crossZoneInstanceCreation = crossZoneInstanceCreation; + } + @Override public void setCapacityBytes(Long capacityBytes) { this.capacityBytes = capacityBytes; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql index 0538e45d1dc..4fcb2b75de5 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42100to42200.sql @@ -25,3 +25,6 @@ CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('router_health_check', 'check_result', ' -- Increase length of scripts_version column to 128 due to md5sum to sha512sum change CALL `cloud`.`IDEMPOTENT_CHANGE_COLUMN`('cloud.domain_router', 'scripts_version', 'scripts_version', 'VARCHAR(128)'); + +-- Add the column cross_zone_instance_creation to cloud.backup_repository. if enabled it means that new Instance can be created on all Zones from Backups on this Repository. +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.backup_repository', 'cross_zone_instance_creation', 'TINYINT(1) DEFAULT NULL COMMENT ''Backup Repository can be used for disaster recovery on another zone'''); diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 48082c1c8a4..b228a9f8ce0 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -53,6 +53,11 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { @Inject private DiskOfferingDao diskOfferingDao; + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return true; + } + @Override public String getName() { return "dummy"; @@ -199,7 +204,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return true; + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return new Pair<>(true, null); } } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index e5f98ad291b..9cd2f20e386 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -70,9 +70,18 @@ import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; + public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + ConfigKey NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class, + "nas.backup.restore.mount.timeout", + "30", + "Timeout in seconds after which backup repository mount for restore fails.", + true, + BackupFrameworkEnabled.key()); + @Inject private BackupDao backupDao; @@ -115,30 +124,45 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private DiskOfferingDao diskOfferingDao; - protected Host getLastVMHypervisorHost(VirtualMachine vm) { - Long hostId = vm.getLastHostId(); - if (hostId == null) { - LOG.debug("Cannot find last host for vm. This should never happen, please check your database."); + private Long getClusterIdFromRootVolume(VirtualMachine vm) { + VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId()); + StoragePoolVO rootDiskPool = primaryDataStoreDao.findById(rootVolume.getPoolId()); + if (rootDiskPool == null) { return null; } - Host host = hostDao.findById(hostId); + return rootDiskPool.getClusterId(); + } - if (host.getStatus() == Status.Up) { - return host; - } else { + protected Host getVMHypervisorHost(VirtualMachine vm) { + Long hostId = vm.getLastHostId(); + Long clusterId = null; + + if (hostId != null) { + Host host = hostDao.findById(hostId); + if (host.getStatus() == Status.Up) { + return host; + } // Try to find any Up host in the same cluster - for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(host.getClusterId())) { + clusterId = host.getClusterId(); + } else { + // Try to find any Up host in the same cluster as the root volume + clusterId = getClusterIdFromRootVolume(vm); + } + + if (clusterId != null) { + for (final Host hostInCluster : hostDao.findHypervisorHostInCluster(clusterId)) { if (hostInCluster.getStatus() == Status.Up) { - LOG.debug("Found Host {} in cluster {}", hostInCluster, host.getClusterId()); + LOG.debug("Found Host {} in cluster {}", hostInCluster, clusterId); return hostInCluster; } } } + // Try to find any Host in the zone - return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, host.getDataCenterId()); + return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, vm.getDataCenterId()); } - protected Host getVMHypervisorHost(VirtualMachine vm) { + protected Host getVMHypervisorHostForBackup(VirtualMachine vm) { Long hostId = vm.getHostId(); if (hostId == null && VirtualMachine.State.Running.equals(vm.getState())) { throw new CloudRuntimeException(String.format("Unable to find the hypervisor host for %s. Make sure the virtual machine is running", vm.getName())); @@ -158,7 +182,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public Pair takeBackup(final VirtualMachine vm, Boolean quiesceVM) { - final Host host = getVMHypervisorHost(vm); + final Host host = getVMHypervisorHostForBackup(vm); final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(vm.getBackupOfferingId()); if (backupRepository == null) { @@ -249,16 +273,16 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { return restoreVMBackup(vm, backup); } @Override public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { - return restoreVMBackup(vm, backup); + return restoreVMBackup(vm, backup).first(); } - private boolean restoreVMBackup(VirtualMachine vm, Backup backup) { + private Pair restoreVMBackup(VirtualMachine vm, Backup backup) { List backedVolumesUUIDs = backup.getBackedUpVolumes().stream() .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId)) .map(Backup.VolumeInfo::getUuid) @@ -271,7 +295,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm, backup); BackupRepository backupRepository = getBackupRepository(backup); - final Host host = getLastVMHypervisorHost(vm); + final Host host = getVMHypervisorHost(vm); RestoreBackupCommand restoreCommand = new RestoreBackupCommand(); restoreCommand.setBackupPath(backup.getExternalId()); restoreCommand.setBackupRepoType(backupRepository.getType()); @@ -282,6 +306,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setRestoreVolumePaths(getVolumePaths(restoreVolumes)); restoreCommand.setVmExists(vm.getRemoved() == null); restoreCommand.setVmState(vm.getState()); + restoreCommand.setMountTimeout(NASBackupRestoreMountTimeout.value()); BackupAnswer answer; try { @@ -291,7 +316,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); } - return answer.getResult(); + return new Pair<>(answer.getResult(), answer.getDetails()); } private List getVolumePaths(List volumes) { @@ -398,7 +423,7 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co final Host host; final VirtualMachine vm = vmInstanceDao.findByIdIncludingRemoved(backup.getVmId()); if (vm != null) { - host = getLastVMHypervisorHost(vm); + host = getVMHypervisorHost(vm); } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } @@ -513,9 +538,19 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co return true; } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backupOffering.getId()); + if (backupRepository == null) { + throw new CloudRuntimeException("Backup repository not found for the backup offering" + backupOffering.getName()); + } + return Boolean.TRUE.equals(backupRepository.crossZoneInstanceCreationEnabled()); + } + @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ + NASBackupRestoreMountTimeout }; } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index d6f29dc1aac..7540cabbbf5 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -21,11 +21,9 @@ import static org.mockito.Mockito.mock; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; -import org.apache.cloudstack.backup.dao.BackupDao; -import org.apache.cloudstack.backup.dao.BackupRepositoryDao; -import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -39,6 +37,7 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.AgentManager; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -51,6 +50,12 @@ import com.cloud.utils.Pair; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupRepositoryDao; +import org.apache.cloudstack.backup.dao.BackupOfferingDao; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; + @RunWith(MockitoJUnitRunner.class) public class NASBackupProviderTest { @Spy @@ -84,6 +89,9 @@ public class NASBackupProviderTest { @Mock private ResourceManager resourceManager; + @Mock + private PrimaryDataStoreDao storagePoolDao; + @Test public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { Long hostId = 1L; @@ -94,7 +102,7 @@ public class NASBackupProviderTest { ReflectionTestUtils.setField(backup, "id", 1L); BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getLastHostId()).thenReturn(hostId); @@ -113,7 +121,7 @@ public class NASBackupProviderTest { @Test public void testSyncBackupStorageStats() throws AgentUnavailableException, OperationTimedoutException { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); HostVO host = mock(HostVO.class); Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, 1L)).thenReturn(host); @@ -132,7 +140,7 @@ public class NASBackupProviderTest { @Test public void testListBackupOfferings() { BackupRepositoryVO backupRepository = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1024L); + "nfs", "address", "sync", 1024L, null); ReflectionTestUtils.setField(backupRepository, "uuid", "uuid"); Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")).thenReturn(Collections.singletonList(backupRepository)); @@ -146,11 +154,11 @@ public class NASBackupProviderTest { @Test public void testGetBackupStorageStats() { BackupRepositoryVO backupRepository1 = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 1000L); + "nfs", "address", "sync", 1000L, null); backupRepository1.setUsedBytes(500L); BackupRepositoryVO backupRepository2 = new BackupRepositoryVO(1L, "nas", "test-repo", - "nfs", "address", "sync", 2000L); + "nfs", "address", "sync", 2000L, null); backupRepository2.setUsedBytes(600L); Mockito.when(backupRepositoryDao.listByZoneAndProvider(1L, "nas")) @@ -227,4 +235,118 @@ public class NASBackupProviderTest { Mockito.verify(backupDao).update(Mockito.anyLong(), Mockito.any(BackupVO.class)); Mockito.verify(agentManager).send(anyLong(), Mockito.any(TakeBackupCommand.class)); } + + @Test + public void testGetVMHypervisorHost() { + Long hostId = 1L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(hostId, result.getId())); + Mockito.verify(hostDao).findById(hostId); + } + + @Test + public void testGetVMHypervisorHostWithHostDown() { + Long hostId = 1L; + Long clusterId = 2L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + + HostVO downHost = mock(HostVO.class); + Mockito.when(downHost.getStatus()).thenReturn(Status.Down); + Mockito.when(downHost.getClusterId()).thenReturn(clusterId); + Mockito.when(hostDao.findById(hostId)).thenReturn(downHost); + + HostVO upHostInCluster = mock(HostVO.class); + Mockito.when(upHostInCluster.getId()).thenReturn(3L); + Mockito.when(upHostInCluster.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(List.of(upHostInCluster)); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(3L), result.getId())); + Mockito.verify(hostDao).findById(hostId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + } + + @Test + public void testGetVMHypervisorHostWithUpHostViaRootVolumeCluster() { + Long vmId = 1L; + Long zoneId = 1L; + Long clusterId = 2L; + Long poolId = 3L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(null); + Mockito.when(vm.getId()).thenReturn(vmId); + + VolumeVO rootVolume = mock(VolumeVO.class); + Mockito.when(rootVolume.getPoolId()).thenReturn(poolId); + Mockito.when(volumeDao.getInstanceRootVolume(vmId)).thenReturn(rootVolume); + + StoragePoolVO storagePool = mock(StoragePoolVO.class); + Mockito.when(storagePool.getClusterId()).thenReturn(clusterId); + Mockito.when(storagePoolDao.findById(poolId)).thenReturn(storagePool); + + HostVO upHostInCluster = mock(HostVO.class); + Mockito.when(upHostInCluster.getId()).thenReturn(4L); + Mockito.when(upHostInCluster.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(List.of(upHostInCluster)); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(4L), result.getId())); + Mockito.verify(volumeDao).getInstanceRootVolume(vmId); + Mockito.verify(storagePoolDao).findById(poolId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + } + + @Test + public void testGetVMHypervisorHostFallbackToZoneWideKVMHost() { + Long hostId = 1L; + Long clusterId = 2L; + Long vmId = 1L; + Long zoneId = 1L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + Mockito.when(vm.getDataCenterId()).thenReturn(zoneId); + + HostVO downHost = mock(HostVO.class); + Mockito.when(downHost.getStatus()).thenReturn(Status.Down); + Mockito.when(downHost.getClusterId()).thenReturn(clusterId); + Mockito.when(hostDao.findById(hostId)).thenReturn(downHost); + + Mockito.when(hostDao.findHypervisorHostInCluster(clusterId)).thenReturn(Collections.emptyList()); + + HostVO fallbackHost = mock(HostVO.class); + Mockito.when(fallbackHost.getId()).thenReturn(5L); + Mockito.when(resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId)) + .thenReturn(fallbackHost); + + Host result = nasBackupProvider.getVMHypervisorHost(vm); + + Assert.assertNotNull(result); + Assert.assertTrue(Objects.equals(Long.valueOf(5L), result.getId())); + Mockito.verify(hostDao).findById(hostId); + Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); + Mockito.verify(resourceManager).findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + } } diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index f39aedb55f2..66b633e11a9 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -158,6 +158,11 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid }; } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + @Override public String getName() { return "networker"; @@ -630,7 +635,7 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid public boolean willDeleteBackupsOnOfferingRemoval() { return false; } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { - return true; + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + return new Pair<>(true, null); } } diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java index c81c5d34ea2..39970dab342 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/VeeamBackupProvider.java @@ -337,11 +337,11 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, } @Override - public boolean restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { + public Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid) { final Long zoneId = backup.getZoneId(); final String restorePointId = backup.getExternalId(); final String restoreLocation = vm.getInstanceName(); - return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid).first(); + return getClient(zoneId).restoreVMToDifferentLocation(restorePointId, restoreLocation, hostIp, dataStoreUuid); } @Override @@ -358,6 +358,11 @@ public class VeeamBackupProvider extends AdapterBase implements BackupProvider, public void syncBackupStorageStats(Long zoneId) { } + @Override + public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { + return false; + } + @Override public String getConfigComponentName() { return BackupService.class.getSimpleName(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 0e5091ebcf4..243cf2efa03 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -61,42 +61,49 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper backedVolumeUUIDs = command.getBackupVolumesUUIDs(); List restoreVolumePaths = command.getRestoreVolumePaths(); String restoreVolumeUuid = command.getRestoreVolumeUUID(); + Integer mountTimeout = command.getMountTimeout() * 1000; String newVolumeId = null; try { + String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions, mountTimeout); if (Objects.isNull(vmExists)) { String volumePath = restoreVolumePaths.get(0); int lastIndex = volumePath.lastIndexOf("/"); newVolumeId = volumePath.substring(lastIndex + 1); - restoreVolume(backupPath, backupRepoType, backupRepoAddress, volumePath, diskType, restoreVolumeUuid, - new Pair<>(vmName, command.getVmState()), mountOptions); + restoreVolume(backupPath, volumePath, diskType, restoreVolumeUuid, + new Pair<>(vmName, command.getVmState()), mountDirectory); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfExistingVM(restoreVolumePaths, backedVolumeUUIDs, backupPath, mountDirectory); } else { - restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, backupRepoType, backupRepoAddress, mountOptions); + restoreVolumesOfDestroyedVMs(restoreVolumePaths, vmName, backupPath, mountDirectory); } } catch (CloudRuntimeException e) { - String errorMessage = "Failed to restore backup for VM: " + vmName + "."; - if (e.getMessage() != null && !e.getMessage().isEmpty()) { - errorMessage += " Details: " + e.getMessage(); - } - logger.error(errorMessage); + String errorMessage = e.getMessage() != null ? e.getMessage() : ""; return new BackupAnswer(command, false, errorMessage); } return new BackupAnswer(command, true, newVolumeId); } - private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { + private void verifyBackupFile(String backupPath, String volUuid) { + if (!checkBackupPathExists(backupPath)) { + throw new CloudRuntimeException(String.format("Backup file for the volume [%s] does not exist.", volUuid)); + } + if (!checkBackupFileImage(backupPath)) { + throw new CloudRuntimeException(String.format("Backup qcow2 file for the volume [%s] is corrupt.", volUuid)); + } + } + + private void restoreVolumesOfExistingVM(List restoreVolumePaths, List backedVolumesUUIDs, + String backupPath, String mountDirectory) { String diskType = "root"; - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); try { for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { String restoreVolumePath = restoreVolumePaths.get(idx); String backupVolumeUuid = backedVolumesUUIDs.get(idx); Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, null, backupPath, diskType, backupVolumeUuid); diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(restoreVolumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -107,15 +114,14 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper volumePaths, String vmName, String backupPath, - String backupRepoType, String backupRepoAddress, String mountOptions) { - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); + private void restoreVolumesOfDestroyedVMs(List volumePaths, String vmName, String backupPath, String mountDirectory) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { String volumePath = volumePaths.get(i); Pair bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, null); diskType = "datadisk"; + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -126,12 +132,12 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountOptions) { - String mountDirectory = mountBackupDirectory(backupRepoAddress, backupRepoType, mountOptions); + private void restoreVolume(String backupPath, String volumePath, String diskType, String volumeUUID, + Pair vmNameAndState, String mountDirectory) { Pair bkpPathAndVolUuid; try { bkpPathAndVolUuid = getBackupPath(mountDirectory, volumePath, backupPath, diskType, volumeUUID); + verifyBackupFile(bkpPathAndVolUuid.first(), bkpPathAndVolUuid.second()); if (!replaceVolumeWithBackup(volumePath, bkpPathAndVolUuid.first())) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", bkpPathAndVolUuid.second())); } @@ -140,8 +146,6 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(bkpPath, volUuid); } + private boolean checkBackupFileImage(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("qemu-img check %s", backupPath)); + return exitValue == 0; + } + + private boolean checkBackupPathExists(String backupPath) { + int exitValue = Script.runSimpleBashScriptForExitValue(String.format("ls %s", backupPath)); + return exitValue == 0; + } + private boolean replaceVolumeWithBackup(String volumePath, String backupPath) { int exitValue = Script.runSimpleBashScriptForExitValue(String.format(RSYNC_COMMAND, backupPath, volumePath)); return exitValue == 0; diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java new file mode 100644 index 00000000000..d120abd0a1b --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -0,0 +1,499 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.script.Script; +import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.backup.BackupAnswer; +import org.apache.cloudstack.backup.RestoreBackupCommand; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtRestoreBackupCommandWrapperTest { + + private LibvirtRestoreBackupCommandWrapper wrapper; + private LibvirtComputingResource libvirtComputingResource; + private RestoreBackupCommand command; + + @Before + public void setUp() { + wrapper = new LibvirtRestoreBackupCommandWrapper(); + libvirtComputingResource = Mockito.mock(LibvirtComputingResource.class); + command = Mockito.mock(RestoreBackupCommand.class); + } + + @Test + public void testExecuteWithVmExistsNull() throws Exception { + when(command.getVmName()).thenReturn("test-vm"); + when(command.getBackupPath()).thenReturn("backup/path"); + when(command.getBackupRepoAddress()).thenReturn("192.168.1.100:/backup"); + when(command.getBackupRepoType()).thenReturn("nfs"); + when(command.getMountOptions()).thenReturn("rw"); + when(command.isVmExists()).thenReturn(null); + when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); + when(command.getRestoreVolumeUUID()).thenReturn("volume-123"); + when(command.getVmState()).thenReturn(VirtualMachine.State.Running); + when(command.getMountTimeout()).thenReturn(30); + + try (MockedStatic filesMock = mockStatic(Files.class)) { + Path tempPath = Mockito.mock(Path.class); + when(tempPath.toString()).thenReturn("/tmp/csbackup.abc123"); + filesMock.when(() -> Files.createTempDirectory(anyString())).thenReturn(tempPath); + + try (MockedStatic + + diff --git a/ui/src/views/storage/RecurringSnapshotVolume.vue b/ui/src/views/storage/RecurringSnapshotVolume.vue index 1cac35c3eba..5eed42a736d 100644 --- a/ui/src/views/storage/RecurringSnapshotVolume.vue +++ b/ui/src/views/storage/RecurringSnapshotVolume.vue @@ -17,11 +17,33 @@ @@ -53,44 +80,150 @@ export default { props: { resource: { type: Object, - required: true + required: false, + default: () => null } }, + inject: ['parentFetchData'], data () { return { loading: false, - dataSource: [] + dataSource: [], + volumes: [], + volumesLoading: false, + selectedVolumeId: null, + selectedVolume: null + } + }, + computed: { + resourceType () { + if (!this.resource) return 'none' + if (this.resource.type === 'ROOT' || this.resource.type === 'DATADISK' || + this.resource.state === 'Ready' || this.resource.state === 'Allocated' || + this.resource.sizegb !== undefined) { + return 'volume' + } + if (this.resource.intervaltype !== undefined || this.resource.schedule !== undefined) { + return 'snapshotpolicy' + } + + return 'unknown' + }, + + isVolumeResource () { + return this.resourceType === 'volume' + }, + + currentVolumeResource () { + if (this.isVolumeResource) { + return this.resource + } else { + return this.selectedVolume + } } }, created () { - this.fetchData() + if (this.isVolumeResource) { + this.fetchData() + } else { + this.fetchVolumes() + } }, methods: { + async fetchVolumes () { + this.volumesLoading = true + try { + const response = await getAPI('listVolumes', { listAll: true }) + const volumes = response.listvolumesresponse.volume || [] + this.volumes = volumes.filter(volume => { + return volume.state === 'Ready' && + (volume.hypervisor !== 'KVM' || + (['Stopped', 'Destroyed'].includes(volume.vmstate)) || + (this.$store.getters.features.kvmsnapshotenabled)) + }) + } catch (error) { + this.$message.error(this.$t('message.error.fetch.volumes')) + console.error('Error fetching volumes:', error) + } finally { + this.volumesLoading = false + } + }, + onVolumeChange (volumeId) { + const volume = this.volumes.find(v => v.id === volumeId) + if (volume) { + this.selectedVolume = volume + this.selectedVolumeId = volumeId + this.dataSource = [] + this.fetchData() + } + }, fetchData () { - const params = {} + const volumeResource = this.currentVolumeResource + if (!volumeResource || !volumeResource.id) { + return + } + + const params = { + volumeid: volumeResource.id, + listAll: true + } + this.dataSource = [] this.loading = true - params.volumeid = this.resource.id + getAPI('listSnapshotPolicies', params).then(json => { this.loading = false const listSnapshotPolicies = json.listsnapshotpoliciesresponse.snapshotpolicy if (listSnapshotPolicies && listSnapshotPolicies.length > 0) { this.dataSource = listSnapshotPolicies } + }).catch(error => { + this.loading = false + this.$message.error(this.$t('message.error.fetch.snapshot.policies')) + console.error('Error fetching snapshot policies:', error) }) }, handleRefresh () { this.fetchData() + this.parentFetchData() }, closeAction () { + this.fetchData() this.$emit('close-action') + }, + filterOption (input, option) { + return option.children[0].children.toLowerCase().indexOf(input.toLowerCase()) >= 0 } } } From 2b1f0bbbdbd6909076e7d7a5cb7fdc0a954450cc Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Fri, 10 Oct 2025 12:35:41 +0530 Subject: [PATCH 065/145] UI: Fix for cluster addition in VMware (#11812) --- ui/src/views/infra/ClusterAdd.vue | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ui/src/views/infra/ClusterAdd.vue b/ui/src/views/infra/ClusterAdd.vue index 7fee1f11bf6..be561832f31 100644 --- a/ui/src/views/infra/ClusterAdd.vue +++ b/ui/src/views/infra/ClusterAdd.vue @@ -155,7 +155,7 @@ - + @@ -266,6 +266,7 @@ export default { this.loading = true getAPI('listZones', { showicon: true }).then(response => { this.zonesList = response.listzonesresponse.zone || [] + this.form.zoneid = this.zonesList?.[0]?.id || null this.fetchPods() }).catch(error => { this.$notifyError(error) @@ -288,7 +289,7 @@ export default { fetchPods () { this.loading = true getAPI('listPods', { - zoneid: this.zoneId + zoneid: this.form.zoneid }).then(response => { this.podsList = response.listpodsresponse.pod || [] }).catch(error => { @@ -314,12 +315,12 @@ export default { this.loading = true this.clustertype = 'ExternalManaged' getAPI('listVmwareDcs', { - zoneid: this.form.zoneId + zoneid: this.form.zoneid }).then(response => { var vmwaredcs = response.listvmwaredcsresponse.VMwareDC if (vmwaredcs !== null) { this.form.host = vmwaredcs[0].vcenter - this.form.dataCenter = vmwaredcs[0].name + this.form.datacenter = vmwaredcs[0].name } }).catch(error => { this.$notification.error({ @@ -352,7 +353,7 @@ export default { var clustername = values.clustername var url = '' if (values.hypervisor === 'VMware') { - clustername = `${this.host}/${this.dataCenter}/${clustername}` + clustername = `${this.form.host}/${this.form.datacenter}/${clustername}` url = `http://${clustername}` } this.loading = true From 67250d99d4494b888270e76202906566bc614673 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Oct 2025 13:09:25 +0530 Subject: [PATCH 066/145] ui: fix add host form state on submit (#11815) --- ui/src/views/infra/HostAdd.vue | 458 +++++++++++++++++---------------- 1 file changed, 230 insertions(+), 228 deletions(-) diff --git a/ui/src/views/infra/HostAdd.vue b/ui/src/views/infra/HostAdd.vue index 4cc179ac467..6879fa89f8b 100644 --- a/ui/src/views/infra/HostAdd.vue +++ b/ui/src/views/infra/HostAdd.vue @@ -17,236 +17,238 @@