diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 01f11b73cd4..4ad1ffb755b 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -524,6 +524,7 @@ public interface UserVmService { * @param userId user ID * @param serviceOffering service offering for the imported VM * @param sshPublicKey ssh key for the imported VM + * @param guestOsId guest OS ID for the imported VM (if not passed, then the guest OS of the template will be used) * @param hostName the name for the imported VM * @param hypervisorType hypervisor type for the imported VM * @param customParameters details for the imported VM @@ -533,7 +534,7 @@ public interface UserVmService { * @throws InsufficientCapacityException in case of errors */ UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceNameInternal, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKey, + final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKey, final Long guestOsId, final String hostName, final HypervisorType hypervisorType, final Map customParameters, final VirtualMachine.PowerState powerState, final LinkedHashMap> networkNicMap) throws InsufficientCapacityException; diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index d244de7115e..41c9a864c9d 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -124,6 +124,9 @@ public interface VirtualMachine extends RunningOn, ControlledEntity, Partition, s_fsm.addTransition(new Transition(State.Stopping, VirtualMachine.Event.StopRequested, State.Stopping, null)); s_fsm.addTransition(new Transition(State.Stopping, VirtualMachine.Event.AgentReportShutdowned, State.Stopped, Arrays.asList(new Impact[]{Impact.USAGE}))); s_fsm.addTransition(new Transition(State.Expunging, VirtualMachine.Event.OperationFailed, State.Expunging,null)); + // Note: In addition to the Stopped -> Error transition for failed VM creation, + // a VM can also transition from Expunging to Error on OperationFailedToError. + s_fsm.addTransition(new Transition(State.Expunging, VirtualMachine.Event.OperationFailedToError, State.Error, null)); s_fsm.addTransition(new Transition(State.Expunging, VirtualMachine.Event.ExpungeOperation, State.Expunging,null)); s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.DestroyRequested, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.ExpungeOperation, State.Expunging, null)); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index f7940460d6c..50ccfbd69c5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; @@ -171,6 +172,13 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "(only for importing VMs from VMware to KVM) optional - if true, forces virt-v2v conversions to write directly on the provided storage pool (avoid using temporary conversion pool).") private Boolean forceConvertToPool; + @Parameter(name = ApiConstants.OS_ID, + type = CommandType.UUID, + entityType = GuestOSResponse.class, + since = "4.22.1", + description = "(only for importing VMs from VMware to KVM) optional - the ID of the guest OS for the imported VM.") + private Long guestOsId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -268,6 +276,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return BooleanUtils.toBooleanDefaultIfNull(forceConvertToPool, false); } + public Long getGuestOsId() { + return guestOsId; + } + @Override public String getEventDescription() { String vmName = getName(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java index 4ff2ebf0a66..d558e4c4f24 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java @@ -45,6 +45,11 @@ public class ListGuestOsCmd extends BaseListCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSResponse.class, description = "List by OS type ID") private Long id; + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, + entityType = GuestOSResponse.class, since = "4.22.1", + description = "Comma separated list of OS types") + private List ids; + @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, description = "List by OS Category ID") private Long osCategoryId; @@ -63,6 +68,10 @@ public class ListGuestOsCmd extends BaseListCmd { return id; } + public List getIds() { + return ids; + } + public Long getOsCategoryId() { return osCategoryId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java index 11930bcbab6..9b5d25aa0dd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java @@ -46,7 +46,6 @@ public class CreateStaticRouteCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.GATEWAY_ID, type = CommandType.UUID, entityType = PrivateGatewayResponse.class, - required = true, description = "The gateway ID we are creating static route for. Mutually exclusive with the nexthop parameter") private Long gatewayId; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java index 195323b741d..3f2dc045e87 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java @@ -51,6 +51,14 @@ public class UnmanagedInstanceResponse extends BaseResponse { @Param(description = "The name of the host to which Instance belongs") private String hostName; + @SerializedName(ApiConstants.HYPERVISOR) + @Param(description = "The hypervisor to which Instance belongs") + private String hypervisor; + + @SerializedName(ApiConstants.HYPERVISOR_VERSION) + @Param(description = "The hypervisor version of the host to which Instance belongs") + private String hypervisorVersion; + @SerializedName(ApiConstants.POWER_STATE) @Param(description = "The power state of the Instance") private String powerState; @@ -140,6 +148,22 @@ public class UnmanagedInstanceResponse extends BaseResponse { this.hostName = hostName; } + public String getHypervisor() { + return hypervisor; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public String getHypervisorVersion() { + return hypervisorVersion; + } + + public void setHypervisorVersion(String hypervisorVersion) { + this.hypervisorVersion = hypervisorVersion; + } + public String getPowerState() { return powerState; } diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java index 5f69f3e46e7..78c598272e0 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -25,11 +25,13 @@ import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.VolumeForImportResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import java.util.Arrays; import java.util.List; -public interface VolumeImportUnmanageService extends PluggableService { +public interface VolumeImportUnmanageService extends PluggableService, Configurable { List SUPPORTED_HYPERVISORS = Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware); @@ -37,6 +39,15 @@ public interface VolumeImportUnmanageService extends PluggableService { List SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem, Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD); + ConfigKey AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class, + "allow.import.volume.with.backing.file", + "Advanced", + "false", + "If enabled, allows QCOW2 volumes with backing files to be imported or unmanaged", + true, + ConfigKey.Scope.Global, + null); + ListResponse listVolumesForImport(ListVolumesForImportCmd cmd); VolumeResponse importVolume(ImportVolumeCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index bba97dff71c..cbb7c4de698 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -55,6 +55,9 @@ public class UnmanagedInstanceTO { private String hostName; + private String hypervisorType; + private String hostHypervisorVersion; + private List disks; private List nics; @@ -168,6 +171,22 @@ public class UnmanagedInstanceTO { this.hostName = hostName; } + public String getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(String hypervisorType) { + this.hypervisorType = hypervisorType; + } + + public String getHostHypervisorVersion() { + return hostHypervisorVersion; + } + + public void setHostHypervisorVersion(String hostHypervisorVersion) { + this.hostHypervisorVersion = hostHypervisorVersion; + } + public List getDisks() { return disks; } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java index b6233b9c270..e1963313eb6 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java @@ -26,6 +26,8 @@ import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware; public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService, PluggableService, Configurable { + String VM_IMPORT_DEFAULT_TEMPLATE_NAME = "system-default-vm-import-dummy-template.iso"; + String KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME = "kvm-default-vm-import-dummy-template"; ConfigKey UnmanageVMPreserveNic = new ConfigKey<>("Advanced", Boolean.class, "unmanage.vm.preserve.nics", "false", "If set to true, do not remove VM nics (and its MAC addresses) when unmanaging a VM, leaving them allocated but not reserved. " + "If set to false, nics are removed and MAC addresses can be reassigned", true, ConfigKey.Scope.Zone); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index b6f31414655..77ea965e50a 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -2326,7 +2326,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra for (final NetworkElement element : networkElements) { if (providersToImplement.contains(element.getProvider())) { if (!_networkModel.isProviderEnabledInPhysicalNetwork(_networkModel.getPhysicalNetworkId(network), element.getProvider().getName())) { - throw new CloudRuntimeException(String.format("Service provider %s either doesn't exist or is not enabled in physical network: %s", element.getProvider().getName(), _physicalNetworkDao.findById(network.getPhysicalNetworkId()))); + throw new CloudRuntimeException(String.format("Service provider %s either doesn't exist or is not enabled in physical network: %s", + element.getProvider().getName(), _physicalNetworkDao.findById(network.getPhysicalNetworkId()))); } if (element instanceof NetworkMigrationResponder) { if (!((NetworkMigrationResponder) element).prepareMigration(profile, network, vm, dest, context)) { @@ -2633,6 +2634,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); guru.deallocate(network, profile, vm); + if (nic.getReservationStrategy() == Nic.ReservationStrategy.Create) { + applyProfileToNicForRelease(nic, profile); + _nicDao.update(nic.getId(), nic); + } if (BooleanUtils.isNotTrue(preserveNics)) { _nicDao.remove(nic.getId()); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java index 1a2b098c40a..f24f3f3b67c 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java @@ -35,7 +35,7 @@ public interface GuestOSDao extends GenericDao { List listByDisplayName(String displayName); - Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay); + Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, List ids, Long osCategoryId, String description, String keyword, Boolean forDisplay); List listIdsByCategoryId(final long categoryId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java index 881be207c1a..09f8772fb59 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java @@ -125,12 +125,12 @@ public class GuestOSDaoImpl extends GenericDaoBase implements G return listBy(sc); } - public Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay) { + public Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, List ids, Long osCategoryId, String description, String keyword, Boolean forDisplay) { final Filter searchFilter = new Filter(GuestOSVO.class, "displayName", true, startIndex, pageSize); final SearchCriteria sc = createSearchCriteria(); - if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); + if (CollectionUtils.isNotEmpty(ids)) { + sc.addAnd("id", SearchCriteria.Op.IN, ids.toArray()); } if (osCategoryId != null) { diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index a03b94faa79..717e3e782f2 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -166,4 +166,12 @@ public interface VolumeDao extends GenericDao, StateDao implements Vol private final SearchBuilder storeAndInstallPathSearch; private final SearchBuilder volumeIdSearch; protected GenericSearchBuilder CountByAccount; + protected final SearchBuilder ExternalUuidSearch; protected GenericSearchBuilder primaryStorageSearch; protected GenericSearchBuilder primaryStorageSearch2; protected GenericSearchBuilder secondaryStorageSearch; @@ -459,6 +460,10 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol CountByAccount.and("idNIN", CountByAccount.entity().getId(), Op.NIN); CountByAccount.done(); + ExternalUuidSearch = createSearchBuilder(); + ExternalUuidSearch.and("externalUuid", ExternalUuidSearch.entity().getExternalUuid(), Op.EQ); + ExternalUuidSearch.done(); + primaryStorageSearch = createSearchBuilder(SumCount.class); primaryStorageSearch.select("sum", Func.SUM, primaryStorageSearch.entity().getSize()); primaryStorageSearch.and("accountId", primaryStorageSearch.entity().getAccountId(), Op.EQ); @@ -934,4 +939,11 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol sc.and(sc.entity().getState(), SearchCriteria.Op.IN, (Object[]) states); return sc.find(); } + + @Override + public VolumeVO findByExternalUuid(String externalUuid) { + SearchCriteria sc = ExternalUuidSearch.create(); + sc.setParameters("externalUuid", externalUuid); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java index c9610f7b9ff..813351d7534 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java @@ -16,6 +16,14 @@ // under the License. package com.cloud.upgrade.dao; +import org.apache.cloudstack.vm.UnmanagedVMsManager; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; + public class Upgrade42200to42210 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @Override @@ -27,4 +35,47 @@ public class Upgrade42200to42210 extends DbUpgradeAbstractImpl implements DbUpgr public String getUpgradedVersion() { return "4.22.1.0"; } + + @Override + public void performDataMigration(Connection conn) { + removeDuplicateKVMImportTemplates(conn); + } + + private void removeDuplicateKVMImportTemplates(Connection conn) { + List templateIds = new ArrayList<>(); + try (PreparedStatement selectStmt = conn.prepareStatement(String.format("SELECT id FROM cloud.vm_template WHERE name='%s' ORDER BY id ASC", UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME))) { + ResultSet rs = selectStmt.executeQuery(); + while (rs.next()) { + templateIds.add(rs.getLong(1)); + } + + if (templateIds.size() <= 1) { + return; + } + + logger.info("Removing duplicate template " + UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME + " entries"); + Long firstTemplateId = templateIds.get(0); + + String updateTemplateSql = "UPDATE cloud.vm_instance SET vm_template_id = ? WHERE vm_template_id = ?"; + String deleteTemplateSql = "DELETE FROM cloud.vm_template WHERE id = ?"; + + try (PreparedStatement updateTemplateStmt = conn.prepareStatement(updateTemplateSql); + PreparedStatement deleteTemplateStmt = conn.prepareStatement(deleteTemplateSql)) { + for (int i = 1; i < templateIds.size(); i++) { + Long duplicateTemplateId = templateIds.get(i); + + // Update VM references + updateTemplateStmt.setLong(1, firstTemplateId); + updateTemplateStmt.setLong(2, duplicateTemplateId); + updateTemplateStmt.executeUpdate(); + + // Delete duplicate dummy template + deleteTemplateStmt.setLong(1, duplicateTemplateId); + deleteTemplateStmt.executeUpdate(); + } + } + } catch (Exception e) { + logger.warn("Failed to remove duplicate template " + UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME + " entries", e); + } + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index cdf903407c1..bbb2b4f3a88 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -67,7 +67,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq; protected SearchBuilder searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; - private SearchBuilder idStateNeqSearch; + private SearchBuilder idStateNinSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; @@ -146,10 +146,10 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotIdWithNonDestroyedState(long snapshotId) { - SearchCriteria sc = idStateNeqSearch.create(); + SearchCriteria sc = idStateNinSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed.name()); return listBy(sc); @@ -488,7 +488,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId) { - SearchCriteria sc = idStateNeqSearch.create(); + SearchCriteria sc = idStateNinSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed.name(), State.Hidden.name()); return listBy(sc); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 54baf226ac4..505c8ef5715 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -18,3 +18,5 @@ --; -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; + +DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index a8a3d3f7bd4..2326a855f32 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -33,3 +33,5 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; -- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; + +UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template'; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql deleted file mode 100644 index 11193c465fd..00000000000 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql +++ /dev/null @@ -1,31 +0,0 @@ --- Licensed to the Apache Software Foundation (ASF) under one --- or more contributor license agreements. See the NOTICE file --- distributed with this work for additional information --- regarding copyright ownership. The ASF licenses this file --- to you under the Apache License, Version 2.0 (the --- "License"); you may not use this file except in compliance --- with the License. You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, --- software distributed under the License is distributed on an --- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --- KIND, either express or implied. See the License for the --- specific language governing permissions and limitations --- under the License. - --- cloud.account_netstats_view source - - -DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; - -CREATE VIEW `cloud`.`account_netstats_view` AS -select - `user_statistics`.`account_id` AS `account_id`, - (sum(`user_statistics`.`net_bytes_received`) + sum(`user_statistics`.`current_bytes_received`)) AS `bytesReceived`, - (sum(`user_statistics`.`net_bytes_sent`) + sum(`user_statistics`.`current_bytes_sent`)) AS `bytesSent` -from - `user_statistics` -group by - `user_statistics`.`account_id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index edc164c40cb..327c6c627e2 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -39,8 +39,8 @@ select `data_center`.`id` AS `data_center_id`, `data_center`.`uuid` AS `data_center_uuid`, `data_center`.`name` AS `data_center_name`, - `account_netstats_view`.`bytesReceived` AS `bytesReceived`, - `account_netstats_view`.`bytesSent` AS `bytesSent`, + `account_netstats`.`bytesReceived` AS `bytesReceived`, + `account_netstats`.`bytesSent` AS `bytesSent`, `vmlimit`.`max` AS `vmLimit`, `vmcount`.`count` AS `vmTotal`, `runningvm`.`vmcount` AS `runningVms`, @@ -89,8 +89,15 @@ from `cloud`.`domain` ON account.domain_id = domain.id left join `cloud`.`data_center` ON account.default_zone_id = data_center.id - left join - `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id + left join lateral ( + select + coalesce(sum(`user_statistics`.`net_bytes_received` + `user_statistics`.`current_bytes_received`), 0) AS `bytesReceived`, + coalesce(sum(`user_statistics`.`net_bytes_sent` + `user_statistics`.`current_bytes_sent`), 0) AS `bytesSent` + from + `cloud`.`user_statistics` + where + `user_statistics`.`account_id` = `account`.`id` + ) AS `account_netstats` ON TRUE left join `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 275f41de43a..bcade3a371c 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -148,8 +148,8 @@ import java.util.HashSet; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; -import static org.apache.cloudstack.vm.UnmanagedVMsManagerImpl.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME; -import static org.apache.cloudstack.vm.UnmanagedVMsManagerImpl.VM_IMPORT_DEFAULT_TEMPLATE_NAME; +import static org.apache.cloudstack.vm.UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME; +import static org.apache.cloudstack.vm.UnmanagedVMsManager.VM_IMPORT_DEFAULT_TEMPLATE_NAME; public class StorageSystemDataMotionStrategy implements DataMotionStrategy { protected Logger logger = LogManager.getLogger(getClass()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java index 114b27d3a5b..c0887415c65 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java @@ -22,6 +22,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import com.cloud.agent.api.Answer; import com.cloud.agent.api.GetRemoteVmsAnswer; import com.cloud.agent.api.GetRemoteVmsCommand; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; @@ -97,6 +98,7 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper, Configurable { @Override public void reorderHostsByPriority(Map priorities, List hosts) { - logger.info("Re-ordering hosts {} by priorities {}", hosts, priorities); + logger.debug("Re-ordering hosts {} by priorities {}", hosts, priorities); hosts.removeIf(host -> DataCenterDeployment.PROHIBITED_HOST_PRIORITY.equals(getHostPriority(priorities, host.getId()))); @@ -1727,7 +1727,7 @@ StateListener, Configurable { } ); - logger.info("Hosts after re-ordering are: {}", hosts); + logger.debug("Hosts after re-ordering are: {}", hosts); } private Integer getHostPriority(Map priorities, Long hostId) { diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index f8c4d1d44d4..15286eaaf54 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -44,6 +44,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import org.apache.cloudstack.acl.ApiKeyPairVO; +import com.cloud.api.query.MutualExclusiveIdsManagerBase; import com.cloud.network.vpc.VpcVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; @@ -842,7 +843,6 @@ import com.cloud.utils.Pair; import com.cloud.utils.PasswordGenerator; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentLifecycle; -import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; @@ -885,7 +885,7 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDetailsDao; -public class ManagementServerImpl extends ManagerBase implements ManagementServer, Configurable { +public class ManagementServerImpl extends MutualExclusiveIdsManagerBase implements ManagementServer, Configurable { protected StateMachine2 _stateMachine; static final String FOR_SYSTEMVMS = "forsystemvms"; @@ -2908,7 +2908,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Override public Pair, Integer> listGuestOSByCriteria(final ListGuestOsCmd cmd) { - final Long id = cmd.getId(); + List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); final Long osCategoryId = cmd.getOsCategoryId(); final String description = cmd.getDescription(); final String keyword = cmd.getKeyword(); @@ -2916,7 +2916,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe final Long pageSize = cmd.getPageSizeVal(); Boolean forDisplay = cmd.getDisplay(); - return _guestOSDao.listGuestOSByCriteria(startIndex, pageSize, id, osCategoryId, description, keyword, forDisplay); + return _guestOSDao.listGuestOSByCriteria(startIndex, pageSize, ids, osCategoryId, description, keyword, forDisplay); } @Override @@ -3048,28 +3048,41 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe throw new InvalidParameterValueException("Hypervisor version parameter cannot be used without specifying a hypervisor : XenServer, KVM or VMware"); } - final SearchCriteria sc = _guestOSHypervisorDao.createSearchCriteria(); + SearchBuilder sb = _guestOSHypervisorDao.createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("guestOsName", sb.entity().getGuestOsName(), SearchCriteria.Op.LIKE); + sb.and("hypervisorType", sb.entity().getHypervisorType(), SearchCriteria.Op.LIKE); + sb.and("hypervisorVersion", sb.entity().getHypervisorVersion(), SearchCriteria.Op.LIKE); + sb.and(guestOsId, sb.entity().getGuestOsId(), SearchCriteria.Op.EQ); + SearchBuilder guestOSSearch = _guestOSDao.createSearchBuilder(); + guestOSSearch.and("display", guestOSSearch.entity().isDisplay(), SearchCriteria.Op.LIKE); + sb.join("guestOSSearch", guestOSSearch, sb.entity().getGuestOsId(), guestOSSearch.entity().getId(), JoinBuilder.JoinType.INNER); + + final SearchCriteria sc = sb.create(); if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); + sc.setParameters("id", SearchCriteria.Op.EQ, id); } if (osTypeId != null) { - sc.addAnd(guestOsId, SearchCriteria.Op.EQ, osTypeId); + sc.setParameters(guestOsId, osTypeId); } if (osNameForHypervisor != null) { - sc.addAnd("guestOsName", SearchCriteria.Op.LIKE, "%" + osNameForHypervisor + "%"); + sc.setParameters("guestOsName", "%" + osNameForHypervisor + "%"); } if (hypervisor != null) { - sc.addAnd("hypervisorType", SearchCriteria.Op.LIKE, "%" + hypervisor + "%"); + sc.setParameters("hypervisorType", "%" + hypervisor + "%"); } if (hypervisorVersion != null) { - sc.addAnd("hypervisorVersion", SearchCriteria.Op.LIKE, "%" + hypervisorVersion + "%"); + sc.setParameters("hypervisorVersion", "%" + hypervisorVersion + "%"); } + // Exclude the mappings for guest OS marked as display = false + sc.setJoinParameters("guestOSSearch", "display", true); + if (osDisplayName != null) { List guestOSVOS = _guestOSDao.listLikeDisplayName(osDisplayName); if (CollectionUtils.isNotEmpty(guestOSVOS)) { diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 67f7128e864..1594e0c9da6 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -2381,6 +2381,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override public TemplateType validateTemplateType(BaseCmd cmd, boolean isAdmin, boolean isCrossZones, HypervisorType hypervisorType) { + if (cmd instanceof GetUploadParamsForIsoCmd) { + return TemplateType.USER; + } if (!(cmd instanceof UpdateTemplateCmd) && !(cmd instanceof RegisterTemplateCmd) && !(cmd instanceof GetUploadParamsForTemplateCmd)) { return null; } diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index c035165a3fa..0a744709644 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -199,4 +199,9 @@ public interface UserVmManager extends UserVmService { Boolean getDestroyRootVolumeOnVmDestruction(Long domainId); + /** + * @return true if the VM is part of a CKS cluster, false otherwise. + */ + boolean isVMPartOfAnyCKSCluster(VMInstanceVO vm); + } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 4597ef81965..f4ea7bd9311 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -1174,21 +1174,21 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir List vmNetworks = _vmNetworkMapDao.getNetworks(vmId); List routers = new ArrayList<>(); //List the stopped routers - for(long vmNetworkId : vmNetworks) { + for (long vmNetworkId : vmNetworks) { List router = _routerDao.listStopped(vmNetworkId); routers.addAll(router); } //A vm may not have many nics attached and even fewer routers might be stopped (only in exceptional cases) //Safe to start the stopped router serially, this is consistent with the way how multiple networks are added to vm during deploy //and routers are started serially ,may revisit to make this process parallel - for(DomainRouterVO routerToStart : routers) { + for (DomainRouterVO routerToStart : routers) { logger.warn("Trying to start router {} as part of vm: {} reboot", routerToStart, vm); _virtualNetAppliance.startRouter(routerToStart.getId(),true); } } } catch (ConcurrentOperationException e) { throw new CloudRuntimeException("Concurrent operations on starting router. " + e); - } catch (Exception ex){ + } catch (Exception ex) { throw new CloudRuntimeException("Router start failed due to" + ex); } finally { if (logger.isInfoEnabled()) { @@ -1473,7 +1473,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir macAddress = validateOrReplaceMacAddress(macAddress, network); - if(_nicDao.findByNetworkIdAndMacAddress(networkId, macAddress) != null) { + if (_nicDao.findByNetworkIdAndMacAddress(networkId, macAddress) != null) { throw new CloudRuntimeException("A NIC with this MAC address exists for network: " + network.getUuid()); } @@ -1499,7 +1499,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir vmInstance, dc, network, dataCenterDao.findById(network.getDataCenterId()))); } - if(_networkModel.getNicInNetwork(vmInstance.getId(),network.getId()) != null){ + if (_networkModel.getNicInNetwork(vmInstance.getId(),network.getId()) != null) { logger.debug("Instance {} already in network {} going to add another NIC", vmInstance, network); } else { //* get all vms hostNames in the network @@ -1527,7 +1527,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } catch (ConcurrentOperationException e) { throw new CloudRuntimeException("Concurrent operations on adding NIC to " + vmInstance + ": " + e); } finally { - if(cleanUp) { + if (cleanUp) { try { _itMgr.removeVmFromNetwork(vmInstance, network, null); } catch (ResourceUnavailableException e) { @@ -2275,7 +2275,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir answer = _agentMgr.easySend(neighbor.getId(), cmd); } - if (answer != null && answer instanceof GetVolumeStatsAnswer){ + if (answer != null && answer instanceof GetVolumeStatsAnswer) { GetVolumeStatsAnswer volstats = (GetVolumeStatsAnswer)answer; if (volstats.getVolumeStats() != null) { volumeStatsByUuid.putAll(volstats.getVolumeStats()); @@ -2286,7 +2286,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return volumeStatsByUuid.size() > 0 ? volumeStatsByUuid : null; } - private List getVolumesByHost(HostVO host, StoragePool pool){ + private List getVolumesByHost(HostVO host, StoragePool pool) { List vmsPerHost = _vmInstanceDao.listByHostId(host.getId()); return vmsPerHost.stream() .flatMap(vm -> _volsDao.findNonDestroyedVolumesByInstanceIdAndPoolId(vm.getId(),pool.getId()).stream().map(vol -> @@ -2558,6 +2558,22 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } + private void transitionExpungingToError(long vmId) { + UserVmVO vm = _vmDao.findById(vmId); + if (vm != null && vm.getState() == State.Expunging) { + try { + boolean transitioned = _itMgr.stateTransitTo(vm, VirtualMachine.Event.OperationFailedToError, null); + if (transitioned) { + logger.info("Transitioned VM [{}] from Expunging to Error after failed expunge", vm.getUuid()); + } else { + logger.warn("Failed to persist transition of VM [{}] from Expunging to Error after failed expunge, possibly due to concurrent update", vm.getUuid()); + } + } catch (NoTransitionException e) { + logger.warn("Failed to transition VM {} to Error state: {}", vm, e.getMessage()); + } + } + } + /** * Release network resources, it was done on vm stop previously. * @param id vm id @@ -2566,7 +2582,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir */ private void releaseNetworkResourcesOnExpunge(long id) throws ConcurrentOperationException, ResourceUnavailableException { final VMInstanceVO vmInstance = _vmDao.findById(id); - if (vmInstance != null){ + if (vmInstance != null) { final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vmInstance); _networkMgr.release(profile, false); } @@ -2831,17 +2847,17 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir ) { if (newCpu > currentCpu) { _resourceLimitMgr.incrementVmCpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, newCpu - currentCpu); - } else if (newCpu > 0 && currentCpu > newCpu){ + } else if (newCpu > 0 && currentCpu > newCpu) { _resourceLimitMgr.decrementVmCpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, currentCpu - newCpu); } if (newMemory > currentMemory) { _resourceLimitMgr.incrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, newMemory - currentMemory); - } else if (newMemory > 0 && currentMemory > newMemory){ + } else if (newMemory > 0 && currentMemory > newMemory) { _resourceLimitMgr.decrementVmMemoryResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, currentMemory - newMemory); } if (newGpu > currentGpu) { _resourceLimitMgr.incrementVmGpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, newGpu - currentGpu); - } else if (newGpu > 0 && currentGpu > newGpu){ + } else if (newGpu > 0 && currentGpu > newGpu) { _resourceLimitMgr.decrementVmGpuResourceCount(owner.getAccountId(), vmInstance.isDisplay(), svcOffering, template, currentGpu - newGpu); } } @@ -2972,7 +2988,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir if (userReadOnlySettings.contains(detailName)) { throw new InvalidParameterValueException("You're not allowed to add or edit the read-only setting: " + detailName); } - if (existingDetails.stream().anyMatch(d -> Objects.equals(d.getName(), detailName) && !d.isDisplay())){ + if (existingDetails.stream().anyMatch(d -> Objects.equals(d.getName(), detailName) && !d.isDisplay())) { throw new InvalidParameterValueException("You're not allowed to add or edit the non-displayable setting: " + detailName); } } @@ -3064,25 +3080,24 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private void saveUsageEvent(UserVmVO vm) { // If vm not destroyed - if( vm.getState() != State.Destroyed && vm.getState() != State.Expunging && vm.getState() != State.Error){ + if (vm.getState() != State.Destroyed && vm.getState() != State.Expunging && vm.getState() != State.Error) { - if(vm.isDisplayVm()){ + if (vm.isDisplayVm()) { //1. Allocated VM Usage Event generateUsageEvent(vm, true, EventTypes.EVENT_VM_CREATE); - if(vm.getState() == State.Running || vm.getState() == State.Stopping){ + if (vm.getState() == State.Running || vm.getState() == State.Stopping) { //2. Running VM Usage Event generateUsageEvent(vm, true, EventTypes.EVENT_VM_START); // 3. Network offering usage generateNetworkUsageForVm(vm, true, EventTypes.EVENT_NETWORK_OFFERING_ASSIGN); } - - }else { + } else { //1. Allocated VM Usage Event generateUsageEvent(vm, true, EventTypes.EVENT_VM_DESTROY); - if(vm.getState() == State.Running || vm.getState() == State.Stopping){ + if (vm.getState() == State.Running || vm.getState() == State.Stopping) { //2. Running VM Usage Event generateUsageEvent(vm, true, EventTypes.EVENT_VM_STOP); @@ -3094,8 +3109,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } - private void generateNetworkUsageForVm(VirtualMachine vm, boolean isDisplay, String eventType){ - + private void generateNetworkUsageForVm(VirtualMachine vm, boolean isDisplay, String eventType) { List nics = _nicDao.listByVmId(vm.getId()); for (NicVO nic : nics) { NetworkVO network = _networkDao.findById(nic.getNetworkId()); @@ -3120,9 +3134,9 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new CloudRuntimeException("Unable to find virtual machine with id " + id); } - if(instanceName != null){ + if (instanceName != null) { VMInstanceVO vmInstance = _vmInstanceDao.findVMByInstanceName(instanceName); - if(vmInstance != null && vmInstance.getId() != id){ + if (vmInstance != null && vmInstance.getId() != id) { throw new CloudRuntimeException("Instance name : " + instanceName + " is not unique"); } } @@ -3264,7 +3278,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir networkIds = networks.stream().map(Network::getId).collect(Collectors.toList()); } } catch (InvalidParameterValueException e) { - if(logger.isDebugEnabled()) { + if (logger.isDebugEnabled()) { logger.debug(e.getMessage(),e); } } @@ -3554,8 +3568,19 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir detachVolumesFromVm(vm, dataVols); UserVm destroyedVm = destroyVm(vmId, expunge); - if (expunge && !expunge(vm)) { - throw new CloudRuntimeException("Failed to expunge vm " + destroyedVm); + if (expunge) { + boolean expunged = false; + String errorMsg = ""; + try { + expunged = expunge(vm); + } catch (RuntimeException e) { + logger.error("Failed to expunge VM [{}] due to: {}", vm, e.getMessage(), e); + errorMsg = e.getMessage(); + } + if (!expunged) { + transitionExpungingToError(vm.getId()); + throw new CloudRuntimeException("Failed to expunge VM " + vm.getUuid() + (StringUtils.isNotBlank(errorMsg) ? " due to: " + errorMsg : "")); + } } autoScaleManager.removeVmFromVmGroup(vmId); @@ -4401,7 +4426,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - if (template.getTemplateType().equals(TemplateType.SYSTEM) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType)) { + if (TemplateType.SYSTEM.equals(template.getTemplateType()) && !CKS_NODE.equals(vmType) && !SHAREDFSVM.equals(vmType)) { throw new InvalidParameterValueException(String.format("Unable to use system template %s to deploy a user vm", template)); } @@ -4796,12 +4821,13 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir private UserVmVO commitUserVm(final boolean isImport, final DataCenter zone, final Host host, final Host lastHost, final VirtualMachineTemplate template, final String hostName, final String displayName, final Account owner, final Long diskOfferingId, final Long diskSize, final String userData, Long userDataId, String userDataDetails, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final String sshPublicKeys, final LinkedHashMap> networkNicMap, + final long accountId, final long userId, final ServiceOffering offering, final boolean isIso, final Long guestOsId, final String sshPublicKeys, final LinkedHashMap> networkNicMap, final long id, final String instanceName, final String uuidName, final HypervisorType hypervisorType, final Map customParameters, final Map> extraDhcpOptionMap, final Map dataDiskTemplateToDiskOfferingMap, final Map userVmOVFPropertiesMap, final VirtualMachine.PowerState powerState, final boolean dynamicScalingEnabled, String vmType, final Long rootDiskOfferingId, String sshkeypairs, List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { - UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, template.getGuestOSId(), offering.isOfferHA(), + Long selectedGuestOsId = guestOsId != null ? guestOsId : template.getGuestOSId(); + UserVmVO vm = new UserVmVO(id, instanceName, displayName, template.getId(), hypervisorType, selectedGuestOsId, offering.isOfferHA(), offering.getLimitCpuUse(), owner.getDomainId(), owner.getId(), userId, offering.getId(), userData, userDataId, userDataDetails, hostName); vm.setUuid(uuidName); vm.setDynamicallyScalable(dynamicScalingEnabled); @@ -4827,8 +4853,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir vm.setIsoId(template.getId()); } - long guestOSId = template.getGuestOSId(); - GuestOSVO guestOS = _guestOSDao.findById(guestOSId); + GuestOSVO guestOS = _guestOSDao.findById(selectedGuestOsId); long guestOSCategoryId = guestOS.getCategoryId(); GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(guestOSCategoryId); if (hypervisorType.equals(HypervisorType.VMware)) { @@ -5086,7 +5111,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir List dataDiskInfoList, Volume volume, Snapshot snapshot) throws InsufficientCapacityException { return commitUserVm(false, zone, null, null, template, hostName, displayName, owner, diskOfferingId, diskSize, userData, userDataId, userDataDetails, isDisplayVm, keyboard, - accountId, userId, offering, isIso, sshPublicKeys, networkNicMap, + accountId, userId, offering, isIso, null, sshPublicKeys, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, extraDhcpOptionMap, dataDiskTemplateToDiskOfferingMap, userVmOVFPropertiesMap, null, dynamicScalingEnabled, vmType, rootDiskOfferingId, sshkeypairs, dataDiskInfoList, volume, snapshot); @@ -5118,7 +5143,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Override - public void generateUsageEvent(VirtualMachine vm, boolean isDisplay, String eventType){ + public void generateUsageEvent(VirtualMachine vm, boolean isDisplay, String eventType) { ServiceOfferingVO serviceOffering = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); if (!serviceOffering.isDynamic()) { UsageEventUtils.publishUsageEvent(eventType, vm.getAccountId(), vm.getDataCenterId(), vm.getId(), @@ -5374,7 +5399,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } // add userdata info into vm profile Nic defaultNic = _networkModel.getDefaultNic(vm.getId()); - if(defaultNic != null) { + if (defaultNic != null) { Network network = _networkModel.getNetwork(defaultNic.getNetworkId()); if (_networkModel.isSharedNetworkWithoutServices(network.getId())) { final String serviceOffering = serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()).getDisplayText(); @@ -5625,7 +5650,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir try { VirtualMachineEntity vmEntity = _orchSrvc.getVirtualMachine(vm.getUuid()); - if(forced) { + if (forced) { status = vmEntity.stopForced(Long.toString(userId)); } else { status = vmEntity.stop(Long.toString(userId)); @@ -5805,7 +5830,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir params = createParameterInParameterMap(params, additionalParams, VirtualMachineProfile.Param.VmPassword, password); } - if(additionalParams.containsKey(VirtualMachineProfile.Param.BootIntoSetup)) { + if (additionalParams.containsKey(VirtualMachineProfile.Param.BootIntoSetup)) { if (! HypervisorType.VMware.equals(vm.getHypervisorType())) { throw new InvalidParameterValueException(ApiConstants.BOOT_INTO_SETUP + " makes no sense for " + vm.getHypervisorType()); } @@ -6283,8 +6308,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } if (!serviceOffering.isDynamic()) { - for(String detail: cmd.getDetails().keySet()) { - if(detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { + for (String detail: cmd.getDetails().keySet()) { + if (detail.equalsIgnoreCase(VmDetailConstants.CPU_NUMBER) || detail.equalsIgnoreCase(VmDetailConstants.CPU_SPEED) || detail.equalsIgnoreCase(VmDetailConstants.MEMORY)) { throw new InvalidParameterValueException("cpuNumber or cpuSpeed or memory should not be specified for static service offering"); } } @@ -6503,8 +6528,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir // check if this templateId has a child ISO List child_templates = _templateDao.listByParentTemplatetId(template.getId()); - for (VMTemplateVO tmpl: child_templates){ - if (tmpl.getFormat() == Storage.ImageFormat.ISO){ + for (VMTemplateVO tmpl: child_templates) { + if (tmpl.getFormat() == Storage.ImageFormat.ISO) { logger.info("MDOV trying to attach disk {} to the VM {}", tmpl, vm); _tmplService.attachIso(tmpl.getId(), vm.getId(), true); } @@ -7161,7 +7186,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir checkIfHostOfVMIsInPrepareForMaintenanceState(vm, "Migrate"); - if(serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.pciDevice.toString()) != null) { + if (serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.pciDevice.toString()) != null) { throw new InvalidParameterValueException("Live Migration of GPU enabled VM is not supported"); } @@ -7776,7 +7801,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw ex; } - if(serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.pciDevice.toString()) != null) { + if (serviceOfferingDetailsDao.findDetail(vm.getServiceOfferingId(), GPU.Keys.pciDevice.toString()) != null) { throw new InvalidParameterValueException("Live Migration of GPU enabled VM is not supported"); } @@ -8753,6 +8778,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir throw new InvalidParameterValueException(String.format("Operation not supported for instance: %s", vm.getName())); } + if (isVMPartOfAnyCKSCluster(vm)) { + throw new UnsupportedServiceException("Cannot restore VM with id = " + vm.getUuid() + + " as it belongs to a CKS cluster. Please remove the VM from the CKS cluster before restoring."); + } _accountMgr.checkAccess(caller, null, true, vm); VMTemplateVO template; @@ -9187,8 +9216,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Long hostId = vm.getHostId() != null ? vm.getHostId() : vm.getLastHostId(); if (hostId != null) { - VolumeInfo volumeInfo = volFactory.getVolume(root.getId()); + // default findById() won't search entries with removed field not null Host host = _hostDao.findById(hostId); + if (host == null) { + logger.warn("Host {} not found", hostId); + return; + } + + VolumeInfo volumeInfo = volFactory.getVolume(root.getId()); final Command cmd; @@ -9491,7 +9526,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Override public UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceNameInternal, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKeys, + final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKeys, final Long guestOsId, final String hostName, final HypervisorType hypervisorType, final Map customParameters, final VirtualMachine.PowerState powerState, final LinkedHashMap> networkNicMap) throws InsufficientCapacityException { return Transaction.execute((TransactionCallbackWithException) status -> { @@ -9517,7 +9552,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir final Boolean dynamicScalingEnabled = checkIfDynamicScalingCanBeEnabled(null, serviceOffering, template, zone.getId()); return commitUserVm(true, zone, host, lastHost, template, hostName, displayName, owner, null, null, userData, null, null, isDisplayVm, keyboard, - accountId, userId, serviceOffering, template.getFormat().equals(ImageFormat.ISO), sshPublicKeys, networkNicMap, + accountId, userId, serviceOffering, template.getFormat().equals(ImageFormat.ISO), guestOsId, sshPublicKeys, networkNicMap, id, instanceName, uuidName, hypervisorType, customParameters, null, null, null, powerState, dynamicScalingEnabled, null, serviceOffering.getDiskOfferingId(), null, null, null, null); }); @@ -9944,10 +9979,15 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - public Boolean getDestroyRootVolumeOnVmDestruction(Long domainId){ + public Boolean getDestroyRootVolumeOnVmDestruction(Long domainId) { return DestroyRootVolumeOnVmDestruction.valueIn(domainId); } + @Override + public boolean isVMPartOfAnyCKSCluster(VMInstanceVO vm) { + return kubernetesServiceHelpers.get(0).findByVmId(vm.getId()) != null; + } + private void setVncPasswordForKvmIfAvailable(Map customParameters, UserVmVO vm) { if (customParameters.containsKey(VmDetailConstants.KVM_VNC_PASSWORD) && StringUtils.isNotEmpty(customParameters.get(VmDetailConstants.KVM_VNC_PASSWORD))) { diff --git a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java index 383644f9aa2..62b5ac38ee2 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageManagerImpl.java @@ -68,6 +68,7 @@ import org.apache.cloudstack.api.response.VolumeForImportResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -394,7 +395,7 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ Map volumeDetails = volume.getDetails(); if (volumeDetails != null && volumeDetails.containsKey(VolumeOnStorageTO.Detail.BACKING_FILE)) { String backingFile = volumeDetails.get(VolumeOnStorageTO.Detail.BACKING_FILE); - if (StringUtils.isNotBlank(backingFile)) { + if (StringUtils.isNotBlank(backingFile) && !AllowImportVolumeWithBackingFile.value()) { logFailureAndThrowException("Volume with backing file cannot be imported or unmanaged."); } } @@ -513,4 +514,16 @@ public class VolumeImportUnmanageManagerImpl implements VolumeImportUnmanageServ volume.setRemoved(new Date()); volumeDao.update(volume.getId(), volume); } + + @Override + public String getConfigComponentName() { + return VolumeImportUnmanageManagerImpl.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[]{ + AllowImportVolumeWithBackingFile + }; + } } diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 14c67417015..e043791c6bf 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -155,8 +155,6 @@ import org.apache.cloudstack.api.command.admin.vm.ListUnmanagedInstancesCmd; import org.apache.cloudstack.api.command.admin.vm.ListVmsForImportCmd; import org.apache.cloudstack.api.command.admin.vm.UnmanageVMInstanceCmd; import org.apache.cloudstack.api.response.ListResponse; -import org.apache.cloudstack.api.response.NicResponse; -import org.apache.cloudstack.api.response.UnmanagedInstanceDiskResponse; import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; @@ -195,15 +193,15 @@ import java.util.stream.Collectors; import static org.apache.cloudstack.api.ApiConstants.MAX_IOPS; import static org.apache.cloudstack.api.ApiConstants.MIN_IOPS; +import static org.apache.cloudstack.storage.volume.VolumeImportUnmanageService.AllowImportVolumeWithBackingFile; import static org.apache.cloudstack.vm.ImportVmTask.Step.CloningInstance; import static org.apache.cloudstack.vm.ImportVmTask.Step.Completed; import static org.apache.cloudstack.vm.ImportVmTask.Step.ConvertingInstance; import static org.apache.cloudstack.vm.ImportVmTask.Step.Importing; public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { - public static final String VM_IMPORT_DEFAULT_TEMPLATE_NAME = "system-default-vm-import-dummy-template.iso"; - public static final String KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME = "kvm-default-vm-import-dummy-template"; protected Logger logger = LogManager.getLogger(UnmanagedVMsManagerImpl.class); + private static final long OTHER_LINUX_64_GUEST_OS_ID = 99; private static final List importUnmanagedInstancesSupportedHypervisors = Arrays.asList(Hypervisor.HypervisorType.VMware, Hypervisor.HypervisorType.KVM); @@ -326,7 +324,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { try { template = VMTemplateVO.createSystemIso(templateDao.getNextInSequence(Long.class, "id"), templateName, templateName, true, "", true, 64, Account.ACCOUNT_ID_SYSTEM, "", - "VM Import Default Template", false, 1); + "VM Import Default Template", false, OTHER_LINUX_64_GUEST_OS_ID); template.setState(VirtualMachineTemplate.State.Inactive); template = templateDao.persist(template); if (template == null) { @@ -340,68 +338,6 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { return template; } - private UnmanagedInstanceResponse createUnmanagedInstanceResponse(UnmanagedInstanceTO instance, Cluster cluster, Host host) { - UnmanagedInstanceResponse response = new UnmanagedInstanceResponse(); - response.setName(instance.getName()); - - if (cluster != null) { - response.setClusterId(cluster.getUuid()); - } - if (host != null) { - response.setHostId(host.getUuid()); - response.setHostName(host.getName()); - } - response.setPowerState(instance.getPowerState().toString()); - response.setCpuCores(instance.getCpuCores()); - response.setCpuSpeed(instance.getCpuSpeed()); - response.setCpuCoresPerSocket(instance.getCpuCoresPerSocket()); - response.setMemory(instance.getMemory()); - response.setOperatingSystemId(instance.getOperatingSystemId()); - response.setOperatingSystem(instance.getOperatingSystem()); - response.setObjectName("unmanagedinstance"); - - if (instance.getDisks() != null) { - for (UnmanagedInstanceTO.Disk disk : instance.getDisks()) { - UnmanagedInstanceDiskResponse diskResponse = new UnmanagedInstanceDiskResponse(); - diskResponse.setDiskId(disk.getDiskId()); - if (StringUtils.isNotEmpty(disk.getLabel())) { - diskResponse.setLabel(disk.getLabel()); - } - diskResponse.setCapacity(disk.getCapacity()); - diskResponse.setController(disk.getController()); - diskResponse.setControllerUnit(disk.getControllerUnit()); - diskResponse.setPosition(disk.getPosition()); - diskResponse.setImagePath(disk.getImagePath()); - diskResponse.setDatastoreName(disk.getDatastoreName()); - diskResponse.setDatastoreHost(disk.getDatastoreHost()); - diskResponse.setDatastorePath(disk.getDatastorePath()); - diskResponse.setDatastoreType(disk.getDatastoreType()); - response.addDisk(diskResponse); - } - } - - if (instance.getNics() != null) { - for (UnmanagedInstanceTO.Nic nic : instance.getNics()) { - NicResponse nicResponse = new NicResponse(); - nicResponse.setId(nic.getNicId()); - nicResponse.setNetworkName(nic.getNetwork()); - nicResponse.setMacAddress(nic.getMacAddress()); - if (StringUtils.isNotEmpty(nic.getAdapterType())) { - nicResponse.setAdapterType(nic.getAdapterType()); - } - if (!CollectionUtils.isEmpty(nic.getIpAddress())) { - nicResponse.setIpAddresses(nic.getIpAddress()); - } - nicResponse.setVlanId(nic.getVlan()); - nicResponse.setIsolatedPvlanId(nic.getPvlan()); - nicResponse.setIsolatedPvlanType(nic.getPvlanType()); - response.addNic(nicResponse); - } - } - - return response; - } - private List getAdditionalNameFilters(Cluster cluster) { List additionalNameFilter = new ArrayList<>(); if (cluster == null) { @@ -1145,7 +1081,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { private UserVm importVirtualMachineInternal(final UnmanagedInstanceTO unmanagedInstance, final String instanceNameInternal, final DataCenter zone, final Cluster cluster, final HostVO host, final VirtualMachineTemplate template, final String displayName, final String hostName, final Account caller, final Account owner, final Long userId, final ServiceOfferingVO serviceOffering, final Map dataDiskOfferingMap, - final Map nicNetworkMap, final Map callerNicIpAddressMap, + final Map nicNetworkMap, final Map callerNicIpAddressMap, final Long guestOsId, final Map details, final boolean migrateAllowed, final boolean forced, final boolean isImportUnmanagedFromSameHypervisor) { logger.debug(LogUtils.logGsonWithoutException("Trying to import VM [%s] with name [%s], in zone [%s], cluster [%s], and host [%s], using template [%s], service offering [%s], disks map [%s], NICs map [%s] and details [%s].", unmanagedInstance, displayName, zone, cluster, host, template, serviceOffering, dataDiskOfferingMap, nicNetworkMap, details)); @@ -1231,7 +1167,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { try { userVm = userVmManager.importVM(zone, host, template, internalCSName, displayName, owner, null, caller, true, null, owner.getAccountId(), userId, - validatedServiceOffering, null, hostName, + validatedServiceOffering, null, guestOsId, hostName, cluster.getHypervisorType(), allDetails, powerState, null); } catch (InsufficientCapacityException ice) { String errorMsg = String.format("Failed to import VM [%s] due to [%s].", displayName, ice.getMessage()); @@ -1550,10 +1486,11 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { protected VMTemplateVO getTemplateForImportInstance(Long templateId, Hypervisor.HypervisorType hypervisorType) { VMTemplateVO template; if (templateId == null) { - String templateName = (Hypervisor.HypervisorType.KVM.equals(hypervisorType)) ? KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME : VM_IMPORT_DEFAULT_TEMPLATE_NAME; + boolean isKVMHypervisor = Hypervisor.HypervisorType.KVM.equals(hypervisorType); + String templateName = (isKVMHypervisor) ? KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME : VM_IMPORT_DEFAULT_TEMPLATE_NAME; template = templateDao.findByName(templateName); if (template == null) { - template = createDefaultDummyVmImportTemplate(Hypervisor.HypervisorType.KVM == hypervisorType); + template = createDefaultDummyVmImportTemplate(isKVMHypervisor); if (template == null) { throw new InvalidParameterValueException(String.format("Default VM import template with unique name: %s for hypervisor: %s cannot be created. Please use templateid parameter for import", templateName, hypervisorType.toString())); } @@ -1633,7 +1570,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { userVm = importVirtualMachineInternal(unmanagedInstance, instanceName, zone, cluster, host, template, displayName, hostName, CallContext.current().getCallingAccount(), owner, userId, serviceOffering, dataDiskOfferingMap, - nicNetworkMap, nicIpAddressMap, + nicNetworkMap, nicIpAddressMap, null, details, migrateAllowed, forced, true); break; } @@ -1713,6 +1650,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { Long convertStoragePoolId = cmd.getConvertStoragePoolId(); String extraParams = cmd.getExtraParams(); boolean forceConvertToPool = cmd.getForceConvertToPool(); + Long guestOsId = cmd.getGuestOsId(); if ((existingVcenterId == null && vcenter == null) || (existingVcenterId != null && vcenter != null)) { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, @@ -1800,7 +1738,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { UserVm userVm = importVirtualMachineInternal(convertedInstance, null, zone, destinationCluster, null, template, displayName, hostName, caller, owner, userId, serviceOffering, dataDiskOfferingMap, - nicNetworkMap, nicIpAddressMap, + nicNetworkMap, nicIpAddressMap, guestOsId, details, false, forced, false); long timeElapsedInSecs = (System.currentTimeMillis() - importStartTime) / 1000; logger.debug(String.format("VMware VM %s imported successfully to CloudStack instance %s (%s), Time taken: %d secs, OVF files imported from %s, Source VMware VM details - OS: %s, PowerState: %s, Disks: %s, NICs: %s", @@ -2366,6 +2304,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { * Perform validations before attempting to unmanage a VM from CloudStack: * - VM must not have any associated volume snapshot * - VM must not have an attached ISO + * - VM must not belong to any CKS cluster + * @throws UnsupportedServiceException in case any of the validations above fail */ void performUnmanageVMInstancePrechecks(VMInstanceVO vmVO) { if (hasVolumeSnapshotsPriorToUnmanageVM(vmVO)) { @@ -2377,6 +2317,11 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { throw new UnsupportedServiceException("Cannot unmanage VM with id = " + vmVO.getUuid() + " as there is an ISO attached. Please detach ISO before unmanaging."); } + + if (userVmManager.isVMPartOfAnyCKSCluster(vmVO)) { + throw new UnsupportedServiceException("Cannot unmanage VM with id = " + vmVO.getUuid() + + " as it belongs to a CKS cluster. Please remove the VM from the CKS cluster before unmanaging."); + } } private boolean hasVolumeSnapshotsPriorToUnmanageVM(VMInstanceVO vmVO) { @@ -2677,7 +2622,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { try { userVm = userVmManager.importVM(zone, null, template, null, displayName, owner, null, caller, true, null, owner.getAccountId(), userId, - serviceOffering, null, hostName, + serviceOffering, null, null, hostName, Hypervisor.HypervisorType.KVM, allDetails, powerState, null); } catch (InsufficientCapacityException ice) { logger.error(String.format("Failed to import vm name: %s", instanceName), ice); @@ -2815,7 +2760,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { try { userVm = userVmManager.importVM(zone, null, template, null, displayName, owner, null, caller, true, null, owner.getAccountId(), userId, - serviceOffering, null, hostName, + serviceOffering, null, null, hostName, Hypervisor.HypervisorType.KVM, allDetails, powerState, networkNicMap); } catch (InsufficientCapacityException ice) { logger.error(String.format("Failed to import vm name: %s", instanceName), ice); @@ -2909,7 +2854,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { } if (volumeDetails.containsKey(VolumeOnStorageTO.Detail.BACKING_FILE)) { String backingFile = volumeDetails.get(VolumeOnStorageTO.Detail.BACKING_FILE); - if (StringUtils.isNotBlank(backingFile)) { + if (StringUtils.isNotBlank(backingFile) && !AllowImportVolumeWithBackingFile.value()) { logFailureAndThrowException("Volume with backing file cannot be imported or unmanaged."); } } @@ -3000,9 +2945,8 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { !instance.getName().toLowerCase().contains(keyword)) { continue; } - responses.add(createUnmanagedInstanceResponse(instance, null, null)); + responses.add(responseGenerator.createUnmanagedInstanceResponse(instance, null, null)); } - ListResponse listResponses = new ListResponse<>(); listResponses.setResponses(responses, responses.size()); return listResponses; diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index f7439386fab..4e6627f4640 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -198,6 +198,7 @@ import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.UUIDManager; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.exception.ExceptionProxyObject; +import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.dao.NicDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDetailsDao; @@ -1501,6 +1502,7 @@ public class UserVmManagerImplTest { when(cmd.getVmId()).thenReturn(vmId); when(cmd.getTemplateId()).thenReturn(2L); when(userVmDao.findById(vmId)).thenReturn(userVmVoMock); + Mockito.doReturn(false).when(userVmManagerImpl).isVMPartOfAnyCKSCluster(userVmVoMock); userVmManagerImpl.restoreVM(cmd); } @@ -4251,4 +4253,56 @@ public class UserVmManagerImplTest { verify(vmInstanceDetailsDao, never()).removeDetailsWithPrefix(anyLong(), anyString()); verify(userVmManagerImpl, never()).addExtraConfig(any(UserVmVO.class), anyString()); } + + @Test + public void testTransitionExpungingToErrorVmInExpungingState() throws Exception { + UserVmVO vm = mock(UserVmVO.class); + when(vm.getState()).thenReturn(VirtualMachine.State.Expunging); + when(vm.getUuid()).thenReturn("test-uuid"); + when(userVmDao.findById(vmId)).thenReturn(vm); + when(virtualMachineManager.stateTransitTo(eq(vm), eq(VirtualMachine.Event.OperationFailedToError), eq(null))).thenReturn(true); + + java.lang.reflect.Method method = UserVmManagerImpl.class.getDeclaredMethod("transitionExpungingToError", long.class); + method.setAccessible(true); + method.invoke(userVmManagerImpl, vmId); + + Mockito.verify(virtualMachineManager).stateTransitTo(vm, VirtualMachine.Event.OperationFailedToError, null); + } + + @Test + public void testTransitionExpungingToErrorVmNotInExpungingState() throws Exception { + UserVmVO vm = mock(UserVmVO.class); + when(vm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(userVmDao.findById(vmId)).thenReturn(vm); + + java.lang.reflect.Method method = UserVmManagerImpl.class.getDeclaredMethod("transitionExpungingToError", long.class); + method.setAccessible(true); + method.invoke(userVmManagerImpl, vmId); + + Mockito.verify(virtualMachineManager, Mockito.never()).stateTransitTo(any(VirtualMachine.class), any(VirtualMachine.Event.class), any()); + } + + @Test + public void testTransitionExpungingToErrorVmNotFound() throws Exception { + when(userVmDao.findById(vmId)).thenReturn(null); + + java.lang.reflect.Method method = UserVmManagerImpl.class.getDeclaredMethod("transitionExpungingToError", long.class); + method.setAccessible(true); + method.invoke(userVmManagerImpl, vmId); + + Mockito.verify(virtualMachineManager, Mockito.never()).stateTransitTo(any(VirtualMachine.class), any(VirtualMachine.Event.class), any()); + } + + @Test + public void testTransitionExpungingToErrorHandlesNoTransitionException() throws Exception { + UserVmVO vm = mock(UserVmVO.class); + when(vm.getState()).thenReturn(VirtualMachine.State.Expunging); + when(userVmDao.findById(vmId)).thenReturn(vm); + when(virtualMachineManager.stateTransitTo(eq(vm), eq(VirtualMachine.Event.OperationFailedToError), eq(null))) + .thenThrow(new NoTransitionException("no transition")); + + java.lang.reflect.Method method = UserVmManagerImpl.class.getDeclaredMethod("transitionExpungingToError", long.class); + method.setAccessible(true); + method.invoke(userVmManagerImpl, vmId); + } } diff --git a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java index a24ba7f068b..98e6388f3d6 100644 --- a/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImplTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.when; import java.net.URI; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.LinkedHashMap; @@ -39,6 +40,9 @@ import java.util.Map; import java.util.UUID; import com.cloud.offering.DiskOffering; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.dao.SnapshotDao; import com.cloud.vm.ImportVMTaskVO; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ResponseGenerator; @@ -241,6 +245,8 @@ public class UnmanagedVMsManagerImplTest { private StoragePoolHostDao storagePoolHostDao; @Mock private ImportVmTasksManager importVmTasksManager; + @Mock + private SnapshotDao snapshotDao; @Mock private VMInstanceVO virtualMachine; @@ -360,7 +366,7 @@ public class UnmanagedVMsManagerImplTest { when(primaryDataStoreDao.listPoolByHostPath(Mockito.anyString(), Mockito.anyString())).thenReturn(pools); when(userVmManager.importVM(nullable(DataCenter.class), nullable(Host.class), nullable(VirtualMachineTemplate.class), nullable(String.class), nullable(String.class), nullable(Account.class), nullable(String.class), nullable(Account.class), nullable(Boolean.class), nullable(String.class), - nullable(Long.class), nullable(Long.class), nullable(ServiceOffering.class), nullable(String.class), + nullable(Long.class), nullable(Long.class), nullable(ServiceOffering.class), nullable(String.class), nullable(Long.class), nullable(String.class), nullable(Hypervisor.HypervisorType.class), nullable(Map.class), nullable(VirtualMachine.PowerState.class), nullable(LinkedHashMap.class))).thenReturn(userVm); NetworkVO networkVO = Mockito.mock(NetworkVO.class); when(networkVO.getGuestType()).thenReturn(Network.GuestType.L2); @@ -568,6 +574,53 @@ public class UnmanagedVMsManagerImplTest { unmanagedVMsManager.unmanageVMInstance(virtualMachineId, null, false); } + @Test(expected = UnsupportedServiceException.class) + public void testUnmanageVMInstanceWithVolumeSnapshotsFail() { + when(virtualMachine.getType()).thenReturn(VirtualMachine.Type.User); + when(virtualMachine.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped); + when(virtualMachine.getId()).thenReturn(virtualMachineId); + UserVmVO userVmVO = mock(UserVmVO.class); + when(userVmDao.findById(anyLong())).thenReturn(userVmVO); + when(vmDao.findById(virtualMachineId)).thenReturn(virtualMachine); + VolumeVO volumeVO = mock(VolumeVO.class); + long volumeId = 20L; + when(volumeVO.getId()).thenReturn(volumeId); + SnapshotVO snapshotVO = mock(SnapshotVO.class); + when(snapshotVO.getState()).thenReturn(Snapshot.State.BackedUp); + when(snapshotDao.listByVolumeId(volumeId)).thenReturn(Collections.singletonList(snapshotVO)); + when(volumeDao.findByInstance(virtualMachineId)).thenReturn(Collections.singletonList(volumeVO)); + unmanagedVMsManager.unmanageVMInstance(virtualMachineId, null, false); + } + + @Test(expected = UnsupportedServiceException.class) + public void testUnmanageVMInstanceWithAssociatedIsoFail() { + when(virtualMachine.getType()).thenReturn(VirtualMachine.Type.User); + when(virtualMachine.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped); + when(virtualMachine.getId()).thenReturn(virtualMachineId); + UserVmVO userVmVO = mock(UserVmVO.class); + when(userVmVO.getIsoId()).thenReturn(null); + when(userVmDao.findById(anyLong())).thenReturn(userVmVO); + when(vmDao.findById(virtualMachineId)).thenReturn(virtualMachine); + when(userVmVO.getIsoId()).thenReturn(1L); + unmanagedVMsManager.unmanageVMInstance(virtualMachineId, null, false); + } + + @Test(expected = UnsupportedServiceException.class) + public void testUnmanageVMInstanceBelongingToCksClusterFail() { + when(virtualMachine.getType()).thenReturn(VirtualMachine.Type.User); + when(virtualMachine.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + when(virtualMachine.getState()).thenReturn(VirtualMachine.State.Stopped); + when(virtualMachine.getId()).thenReturn(virtualMachineId); + UserVmVO userVmVO = mock(UserVmVO.class); + when(userVmVO.getIsoId()).thenReturn(null); + when(userVmDao.findById(anyLong())).thenReturn(userVmVO); + when(vmDao.findById(virtualMachineId)).thenReturn(virtualMachine); + when(userVmManager.isVMPartOfAnyCKSCluster(virtualMachine)).thenReturn(true); + unmanagedVMsManager.unmanageVMInstance(virtualMachineId, null, false); + } + @Test public void testListRemoteInstancesTest() { ListVmsForImportCmd cmd = Mockito.mock(ListVmsForImportCmd.class); diff --git a/test/integration/smoke/test_secondary_storage.py b/test/integration/smoke/test_secondary_storage.py index 4b26950ea64..2ed60a84425 100644 --- a/test/integration/smoke/test_secondary_storage.py +++ b/test/integration/smoke/test_secondary_storage.py @@ -136,7 +136,11 @@ class TestSecStorageServices(cloudstackTestCase): 'Up', "Check state of primary storage pools is Up or not" ) - for _ in range(2): + # Poll until all SSVMs are Running, or timeout after 3 minutes + timeout = 180 + interval = 15 + list_ssvm_response = [] + while timeout > 0: list_ssvm_response = list_ssvms( self.apiclient, systemvmtype='secondarystoragevm', @@ -154,10 +158,12 @@ class TestSecStorageServices(cloudstackTestCase): "Check list System VMs response" ) - for ssvm in list_ssvm_response: - if ssvm.state != 'Running': - time.sleep(30) - continue + if all(ssvm.state == 'Running' for ssvm in list_ssvm_response): + break + + time.sleep(interval) + timeout -= interval + for ssvm in list_ssvm_response: self.assertEqual( ssvm.state, diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 513dfcdaa36..b36e8c1ef1a 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1757,6 +1757,7 @@ "label.no.data": "No data to show", "label.no.errors": "No recent errors", "label.no.items": "No available Items", +"label.no.matching.guest.os.vmware.import": "No matching guest OS mapping found, using the default import template guest OS", "label.no.matching.offering": "No matching offering found", "label.no.matching.network": "No matching Networks found", "label.node.version": "Node version", diff --git a/ui/src/permission.js b/ui/src/permission.js index 671d6626b93..0b87de92c6b 100644 --- a/ui/src/permission.js +++ b/ui/src/permission.js @@ -93,6 +93,7 @@ router.beforeEach((to, from, next) => { return } store.commit('SET_LOGIN_FLAG', true) + store.commit('SET_MS_ID', Cookies.get('managementserverid')) } // store already loaded if (store.getters.passwordChangeRequired) { diff --git a/ui/src/views/compute/DeployVM.vue b/ui/src/views/compute/DeployVM.vue index 26176e76005..474c3bb01ac 100644 --- a/ui/src/views/compute/DeployVM.vue +++ b/ui/src/views/compute/DeployVM.vue @@ -459,6 +459,9 @@ :value="securitygroupids" :loading="loading.networks" :preFillContent="dataPreFill" + :domainId="owner.domainid" + :account="owner.account" + :projectId="owner.projectid" @select-security-group-item="($event) => updateSecurityGroups($event)"> @@ -1501,6 +1504,9 @@ export default { return tabList }, showSecurityGroupSection () { + if (this.zone && this.zone.networktype === 'Basic') { + return true + } if (this.networks.length < 1) { return false } diff --git a/ui/src/views/compute/wizard/SecurityGroupSelection.vue b/ui/src/views/compute/wizard/SecurityGroupSelection.vue index 9d91cc1cbe1..0ea08ec8887 100644 --- a/ui/src/views/compute/wizard/SecurityGroupSelection.vue +++ b/ui/src/views/compute/wizard/SecurityGroupSelection.vue @@ -75,6 +75,18 @@ export default { preFillContent: { type: Object, default: () => {} + }, + domainId: { + type: String, + default: () => '' + }, + account: { + type: String, + default: () => '' + }, + projectId: { + type: String, + default: () => '' } }, data () { @@ -102,6 +114,9 @@ export default { } }, computed: { + ownerParams () { + return `${this.domainId}-${this.account}-${this.projectId}` + }, rowSelection () { return { type: 'checkbox', @@ -121,6 +136,11 @@ export default { this.selectedRowKeys = newValue } }, + ownerParams () { + this.selectedRowKeys = [] + this.$emit('select-security-group-item', null) + this.fetchData() + }, loading () { if (!this.loading) { if (this.preFillContent.securitygroupids) { @@ -140,9 +160,9 @@ export default { methods: { fetchData () { const params = { - projectid: this.$store.getters.project ? this.$store.getters.project.id : null, - domainid: this.$store.getters.project && this.$store.getters.project.id ? null : this.$store.getters.userInfo.domainid, - account: this.$store.getters.project && this.$store.getters.project.id ? null : this.$store.getters.userInfo.account, + projectid: this.projectId || (this.$store.getters.project ? this.$store.getters.project.id : null), + domainid: this.projectId || (this.$store.getters.project && this.$store.getters.project.id) ? null : (this.domainId || this.$store.getters.userInfo.domainid), + account: this.projectId || (this.$store.getters.project && this.$store.getters.project.id) ? null : (this.account || this.$store.getters.userInfo.account), page: this.page, pageSize: this.pageSize } diff --git a/ui/src/views/image/TemplateZones.vue b/ui/src/views/image/TemplateZones.vue index e517aad5167..31b65e556a4 100644 --- a/ui/src/views/image/TemplateZones.vue +++ b/ui/src/views/image/TemplateZones.vue @@ -91,7 +91,7 @@ :rowKey="record => record.datastoreId">