From 19aacad46bb0d58e5d42e56067eddf1f07e0016e Mon Sep 17 00:00:00 2001 From: dahn Date: Tue, 15 Aug 2023 13:51:53 +0200 Subject: [PATCH 01/31] UI: Show iso urls (#7869) * add url to details * some cleanup --- .../main/java/com/cloud/api/query/QueryManagerImpl.java | 8 ++++---- .../main/java/com/cloud/api/query/ViewResponseHelper.java | 2 +- ui/src/config/section/image.js | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 74367a09c53..c28865943a9 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -3940,7 +3940,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Override public ListResponse listIsos(ListIsosCmd cmd) { Pair, Integer> result = searchForIsosInternal(cmd); - ListResponse response = new ListResponse(); + ListResponse response = new ListResponse<>(); ResponseView respView = ResponseView.Restricted; if (cmd instanceof ListIsosCmdByAdmin) { @@ -3967,11 +3967,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q listAll = true; } - List permittedAccountIds = new ArrayList(); - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); + List permittedAccountIds = new ArrayList<>(); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); _accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccountIds, domainIdRecursiveListProject, listAll, false); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); - List permittedAccounts = new ArrayList(); + List permittedAccounts = new ArrayList<>(); for (Long accountId : permittedAccountIds) { permittedAccounts.add(_accountMgr.getAccount(accountId)); } diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index ecfda39972e..8fd42ea3c41 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -610,7 +610,7 @@ public class ViewResponseHelper { } public static List createIsoResponse(ResponseView view, TemplateJoinVO... templates) { - Hashtable vrDataList = new Hashtable(); + Hashtable vrDataList = new Hashtable<>(); for (TemplateJoinVO vr : templates) { TemplateResponse vrData = vrDataList.get(vr.getTempZonePair()); if (vrData == null) { diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index bf1083575dd..c41ab9307a6 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -213,7 +213,7 @@ export default { } return fields }, - details: ['name', 'id', 'displaytext', 'checksum', 'ostypename', 'size', 'bootable', 'isready', 'directdownload', 'isextractable', 'ispublic', 'isfeatured', 'crosszones', 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy'], + details: ['name', 'id', 'displaytext', 'checksum', 'ostypename', 'size', 'bootable', 'isready', 'directdownload', 'isextractable', 'ispublic', 'isfeatured', 'crosszones', 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'url'], searchFilters: ['name', 'zoneid', 'tags'], related: [{ name: 'vm', From cf249f64a3b54b2e5f8ea09282efb1a4bd976bf8 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 16 Aug 2023 08:31:32 +0200 Subject: [PATCH 02/31] api: Force-stop descriptions (#7866) --- .../api/command/admin/internallb/StopInternalLBVMCmd.java | 2 +- .../cloudstack/api/command/admin/router/StopRouterCmd.java | 2 +- .../cloudstack/api/command/admin/systemvm/StopSystemVmCmd.java | 2 +- .../org/apache/cloudstack/api/command/user/vm/StopVMCmd.java | 3 ++- .../main/java/com/cloud/api/commands/StopNetScalerVMCmd.java | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/StopInternalLBVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/StopInternalLBVMCmd.java index 1381f670527..76ad4d438d2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/StopInternalLBVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/internallb/StopInternalLBVMCmd.java @@ -51,7 +51,7 @@ public class StopInternalLBVMCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = DomainRouterResponse.class, required = true, description = "the ID of the internal lb vm") private Long id; - @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM. The caller knows the VM is stopped.") + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM (vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted). To be used if the caller knows the VM is stopped and should be marked as such.") private Boolean forced; // /////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/StopRouterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/StopRouterCmd.java index b9e15661268..2da38d90426 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/StopRouterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/StopRouterCmd.java @@ -49,7 +49,7 @@ public class StopRouterCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = DomainRouterResponse.class, required = true, description = "the ID of the router") private Long id; - @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM. The caller knows the VM is stopped.") + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM (vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted). To be used if the caller knows the VM is stopped and should be marked as such.") private Boolean forced; // /////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/StopSystemVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/StopSystemVmCmd.java index 755c8e46205..4bb533ce5b6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/StopSystemVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/StopSystemVmCmd.java @@ -54,7 +54,7 @@ public class StopSystemVmCmd extends BaseAsyncCmd { description = "The ID of the system virtual machine") private Long id; - @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM. The caller knows the VM is stopped.") + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM (vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted). To be used if the caller knows the VM is stopped and should be marked as such.") private Boolean forced; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java index 529e09a0753..113ba9ed25d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/StopVMCmd.java @@ -55,7 +55,8 @@ public class StopVMCmd extends BaseAsyncCmd implements UserCmd { private Long id; @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM " - + "(vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted). The caller knows the VM is stopped.") + + "(vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted)." + + " This option is to be used if the caller knows the VM is stopped and should be marked as such.") private Boolean forced; // /////////////////////////////////////////////////// diff --git a/plugins/network-elements/netscaler/src/main/java/com/cloud/api/commands/StopNetScalerVMCmd.java b/plugins/network-elements/netscaler/src/main/java/com/cloud/api/commands/StopNetScalerVMCmd.java index 5d754168e49..288e867277a 100644 --- a/plugins/network-elements/netscaler/src/main/java/com/cloud/api/commands/StopNetScalerVMCmd.java +++ b/plugins/network-elements/netscaler/src/main/java/com/cloud/api/commands/StopNetScalerVMCmd.java @@ -56,7 +56,7 @@ public class StopNetScalerVMCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = DomainRouterResponse.class, required = true, description = "the ID of the NetScaler vm") private Long id; - @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM. The caller knows the VM is stopped.") + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, required = false, description = "Force stop the VM (vm is marked as Stopped even when command fails to be send to the backend, otherwise a force poweroff is attempted). To be used if the caller knows the VM is stopped and should be marked as such.") private Boolean forced; // /////////////////////////////////////////////////// From e8b491177f16882879bbcff4bfd7022dd61bae67 Mon Sep 17 00:00:00 2001 From: Charles Queiroz Date: Wed, 16 Aug 2023 07:58:41 +0100 Subject: [PATCH 03/31] server: Replace Hashtable with LinkedHashMap in createIsoResponse (#7844) * Replace Hashtable with LinkedHashMap in createIsoResponse This change replaces the use of Hashtable with LinkedHashMap in the `createIsoResponse` method of `ViewResponseHelper`. The reason for this modification is to maintain the insertion order of entries, which isn't the case with Hashtable. This could lead to more predictable results and behaviors in calling methods. * Replace Hashtable with LinkedHashMap in view response creation methods Changed Hashtable to LinkedHashMap in various response creation methods within ViewResponseHelper class. This modification ensures an ordered iteration which is beneficial for scenarios where the insertion order of responses needs to be maintained consistently. --------- Co-authored-by: Sina Kashipazha --- .../cloud/api/query/ViewResponseHelper.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index 8fd42ea3c41..48031425bb8 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; -import java.util.Hashtable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -150,7 +149,7 @@ public class ViewResponseHelper { public static List createUserVmResponse(ResponseView view, String objectName, Set details, Boolean accumulateStats, Boolean showUserData, UserVmJoinVO... userVms) { Account caller = CallContext.current().getCallingAccount(); - Hashtable vmDataList = new Hashtable(); + LinkedHashMap vmDataList = new LinkedHashMap<>(); // Initialise the vmdatalist with the input data for (UserVmJoinVO userVm : userVms) { @@ -169,7 +168,7 @@ public class ViewResponseHelper { public static List createDomainRouterResponse(DomainRouterJoinVO... routers) { Account caller = CallContext.current().getCallingAccount(); - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (DomainRouterJoinVO vr : routers) { DomainRouterResponse vrData = vrDataList.get(vr.getId()); @@ -187,7 +186,7 @@ public class ViewResponseHelper { public static List createSecurityGroupResponses(List securityGroups) { Account caller = CallContext.current().getCallingAccount(); - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (SecurityGroupJoinVO vr : securityGroups) { SecurityGroupResponse vrData = vrDataList.get(vr.getId()); @@ -205,7 +204,7 @@ public class ViewResponseHelper { } public static List createProjectResponse(EnumSet details, ProjectJoinVO... projects) { - Hashtable prjDataList = new Hashtable(); + LinkedHashMap prjDataList = new LinkedHashMap<>(); // Initialise the prjdatalist with the input data for (ProjectJoinVO p : projects) { ProjectResponse pData = prjDataList.get(p.getId()); @@ -247,7 +246,7 @@ public class ViewResponseHelper { } public static List createHostResponse(EnumSet details, HostJoinVO... hosts) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (HostJoinVO vr : hosts) { HostResponse vrData = ApiDBUtils.newHostResponse(vr, details); @@ -257,7 +256,7 @@ public class ViewResponseHelper { } public static List createHostForMigrationResponse(EnumSet details, HostJoinVO... hosts) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (HostJoinVO vr : hosts) { HostForMigrationResponse vrData = ApiDBUtils.newHostForMigrationResponse(vr, details); @@ -267,7 +266,7 @@ public class ViewResponseHelper { } public static List createVolumeResponse(ResponseView view, VolumeJoinVO... volumes) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); DecimalFormat df = new DecimalFormat("0.0%"); for (VolumeJoinVO vr : volumes) { VolumeResponse vrData = vrDataList.get(vr.getId()); @@ -308,7 +307,7 @@ public class ViewResponseHelper { } public static List createStoragePoolResponse(StoragePoolJoinVO... pools) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (StoragePoolJoinVO vr : pools) { StoragePoolResponse vrData = vrDataList.get(vr.getId()); @@ -345,7 +344,7 @@ public class ViewResponseHelper { } public static List createImageStoreResponse(ImageStoreJoinVO... stores) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (ImageStoreJoinVO vr : stores) { ImageStoreResponse vrData = vrDataList.get(vr.getId()); @@ -362,7 +361,7 @@ public class ViewResponseHelper { } public static List createStoragePoolForMigrationResponse(StoragePoolJoinVO... pools) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); // Initialise the vrdatalist with the input data for (StoragePoolJoinVO vr : pools) { StoragePoolResponse vrData = vrDataList.get(vr.getId()); @@ -577,7 +576,7 @@ public class ViewResponseHelper { } public static List createTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO... templates) { - LinkedHashMap vrDataList = new LinkedHashMap(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); for (TemplateJoinVO vr : templates) { TemplateResponse vrData = vrDataList.get(vr.getTempZonePair()); if (vrData == null) { @@ -594,7 +593,7 @@ public class ViewResponseHelper { } public static List createTemplateUpdateResponse(ResponseView view, TemplateJoinVO... templates) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); for (TemplateJoinVO vr : templates) { TemplateResponse vrData = vrDataList.get(vr.getId()); if (vrData == null) { @@ -610,7 +609,7 @@ public class ViewResponseHelper { } public static List createIsoResponse(ResponseView view, TemplateJoinVO... templates) { - Hashtable vrDataList = new Hashtable<>(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); for (TemplateJoinVO vr : templates) { TemplateResponse vrData = vrDataList.get(vr.getTempZonePair()); if (vrData == null) { @@ -626,7 +625,7 @@ public class ViewResponseHelper { } public static List createAffinityGroupResponses(List groups) { - Hashtable vrDataList = new Hashtable(); + LinkedHashMap vrDataList = new LinkedHashMap<>(); for (AffinityGroupJoinVO vr : groups) { AffinityGroupResponse vrData = vrDataList.get(vr.getId()); if (vrData == null) { From fe70f4d801c39776d5e1dbd262247cd159b2cb04 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Wed, 16 Aug 2023 12:35:18 +0530 Subject: [PATCH 04/31] Remove powermock from services (#7860) --- ...onStrictHostAntiAffinityProcessorTest.java | 4 +- .../src/test/java/common/ClientTest.java | 8 ++-- .../org.mockito.plugins.MockMaker | 1 + .../ConsoleProxyHttpHandlerHelperTest.java | 33 ++++++------- .../org.mockito.plugins.MockMaker | 1 + .../NfsSecondaryStorageResourceTest.java | 46 +++++++++---------- .../org.mockito.plugins.MockMaker | 1 + 7 files changed, 45 insertions(+), 49 deletions(-) create mode 100644 services/console-proxy/rdpconsole/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker create mode 100644 services/console-proxy/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker create mode 100644 services/secondary-storage/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/plugins/affinity-group-processors/non-strict-host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAntiAffinityProcessorTest.java b/plugins/affinity-group-processors/non-strict-host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAntiAffinityProcessorTest.java index 5ea423b374b..7239594fe2c 100644 --- a/plugins/affinity-group-processors/non-strict-host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAntiAffinityProcessorTest.java +++ b/plugins/affinity-group-processors/non-strict-host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAntiAffinityProcessorTest.java @@ -35,7 +35,7 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.Spy; -import org.powermock.modules.junit4.PowerMockRunner; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; import java.util.ArrayList; @@ -47,7 +47,7 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.nullable; import static org.mockito.Mockito.when; -@RunWith(PowerMockRunner.class) +@RunWith(MockitoJUnitRunner.class) public class NonStrictHostAntiAffinityProcessorTest { @Spy diff --git a/services/console-proxy/rdpconsole/src/test/java/common/ClientTest.java b/services/console-proxy/rdpconsole/src/test/java/common/ClientTest.java index 3f5aa909db0..a54b52e78e1 100644 --- a/services/console-proxy/rdpconsole/src/test/java/common/ClientTest.java +++ b/services/console-proxy/rdpconsole/src/test/java/common/ClientTest.java @@ -21,20 +21,20 @@ import static org.mockito.Mockito.when; import org.junit.Test; import org.junit.runner.RunWith; -import org.powermock.modules.junit4.PowerMockRunner; -import org.powermock.reflect.Whitebox; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; import streamer.Element; import streamer.SocketWrapper; -@RunWith(PowerMockRunner.class) +@RunWith(MockitoJUnitRunner.class) public class ClientTest { @Test(expected = NullPointerException.class) public void testAssemblePipelineWhenMainElementIsNull() throws Exception { SocketWrapper socketMock = mock(SocketWrapper.class); when(socketMock.getId()).thenReturn("socket"); - Whitebox.setInternalState(Client.class, "socket", socketMock); + ReflectionTestUtils.setField(Client.class, "socket", socketMock); Element main = null; Client.assemblePipeline(main); diff --git a/services/console-proxy/rdpconsole/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/services/console-proxy/rdpconsole/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/services/console-proxy/rdpconsole/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/services/console-proxy/server/src/test/java/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelperTest.java b/services/console-proxy/server/src/test/java/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelperTest.java index 690b0c19a9e..53389ba36b2 100644 --- a/services/console-proxy/server/src/test/java/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelperTest.java +++ b/services/console-proxy/server/src/test/java/com/cloud/consoleproxy/ConsoleProxyHttpHandlerHelperTest.java @@ -19,33 +19,30 @@ package com.cloud.consoleproxy; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mock; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import org.mockito.junit.MockitoJUnitRunner; import java.util.Map; -@RunWith(PowerMockRunner.class) +@RunWith(MockitoJUnitRunner.class) public class ConsoleProxyHttpHandlerHelperTest { - @Mock - ConsoleProxyPasswordBasedEncryptor encryptor; - @Test - @PrepareForTest({ConsoleProxy.class, ConsoleProxyHttpHandlerHelper.class}) public void testQueryMapExtraParameter() throws Exception { - PowerMockito.mockStatic(ConsoleProxy.class); - PowerMockito.when(ConsoleProxy.getEncryptorPassword()).thenReturn("password"); - PowerMockito.whenNew(ConsoleProxyPasswordBasedEncryptor.class).withArguments(Mockito.anyString()).thenReturn(encryptor); - Mockito.when(encryptor.decryptObject(Mockito.eq(ConsoleProxyClientParam.class), Mockito.anyString())).thenReturn(null); + try (MockedStatic ignore = Mockito.mockStatic(ConsoleProxy.class); + MockedConstruction ignored = Mockito.mockConstruction(ConsoleProxyPasswordBasedEncryptor.class, (mock, context) -> { + Mockito.when(mock.decryptObject(Mockito.eq(ConsoleProxyClientParam.class), Mockito.anyString())).thenReturn(null); + });) { + Mockito.when(ConsoleProxy.getEncryptorPassword()).thenReturn("password"); - String extraValidationToken = "test-token"; - String query = String.format("token=SOME_TOKEN&extra=%s", extraValidationToken); + String extraValidationToken = "test-token"; + String query = String.format("token=SOME_TOKEN&extra=%s", extraValidationToken); - Map queryMap = ConsoleProxyHttpHandlerHelper.getQueryMap(query); - Assert.assertTrue(queryMap.containsKey("extra")); - Assert.assertEquals(extraValidationToken, queryMap.get("extra")); + Map queryMap = ConsoleProxyHttpHandlerHelper.getQueryMap(query); + Assert.assertTrue(queryMap.containsKey("extra")); + Assert.assertEquals(extraValidationToken, queryMap.get("extra")); + } } } diff --git a/services/console-proxy/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/services/console-proxy/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/services/console-proxy/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index 96afcce747b..2aac2766cf7 100644 --- a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -22,54 +22,50 @@ import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileWriter; -import java.io.StringWriter; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.log4j.Level; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; -import org.powermock.api.mockito.PowerMockito; -import org.powermock.core.classloader.annotations.PowerMockIgnore; -import org.powermock.core.classloader.annotations.PrepareForTest; -import org.powermock.modules.junit4.PowerMockRunner; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import com.cloud.test.TestAppender; -@RunWith(PowerMockRunner.class) -@PowerMockIgnore({ "javax.xml.*", "org.xml.*"}) +@RunWith(MockitoJUnitRunner.class) public class NfsSecondaryStorageResourceTest { + @Spy private NfsSecondaryStorageResource resource; - @Before - public void setUp() { - resource = new NfsSecondaryStorageResource(); - } - @Test - @PrepareForTest(NfsSecondaryStorageResource.class) public void testSwiftWriteMetadataFile() throws Exception { - String filename = "testfile"; + String metaFileName = "test_metadata_file"; try { - String expected = "uniquename=test\nfilename=" + filename + "\nsize=100\nvirtualsize=1000"; + String uniqueName = "test_unique_name"; + String filename = "test_filename"; + long size = 1024L; + long virtualSize = 2048L; - StringWriter stringWriter = new StringWriter(); - BufferedWriter bufferWriter = new BufferedWriter(stringWriter); - PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter); + File metaFile = resource.swiftWriteMetadataFile(metaFileName, uniqueName, filename, size, virtualSize); - resource.swiftWriteMetadataFile(filename, "test", filename, 100, 1000); + Assert.assertTrue(metaFile.exists()); + Assert.assertEquals(metaFileName, metaFile.getName()); - Assert.assertEquals(expected, stringWriter.toString()); + String expectedContent = "uniquename=" + uniqueName + "\n" + + "filename=" + filename + "\n" + + "size=" + size + "\n" + + "virtualsize=" + virtualSize; + + String actualContent = new String(java.nio.file.Files.readAllBytes(metaFile.toPath())); + Assert.assertEquals(expectedContent, actualContent); } finally { - File remnance = new File(filename); - remnance.delete(); + File metaFile = new File(metaFileName); + metaFile.delete(); } } diff --git a/services/secondary-storage/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/services/secondary-storage/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/services/secondary-storage/server/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline From 8b5ba13b817e7d53beb0e0012c27d065e96a708c Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Wed, 16 Aug 2023 12:23:24 -0300 Subject: [PATCH 05/31] plugins: Add Custom hypervisor minimal changes (#7692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Description Design document: https://cwiki.apache.org/confluence/display/CLOUDSTACK/%5BDRAFT%5D+Minimal+changes+to+allow+new+dynamic+hypervisor+type%3A+Custom+Hypervisor This PR introduces the minimal changes to add a new hypervisor type (internally named Custom in the codebase, and configurable display name), allowing to write an external hypervisor plugin as a Custom Hypervisor to CloudStack The custom hypervisor name is set by the setting: 'hypervisor.custom.display.name'. The new hypervisor type does not affect the behaviour of any CloudStack operation, it simply introduces a new hypervisor type into the system. CloudStack does not have any means to dynamically add new hypervisor types. The hypervisor types are internally preset by an enum defined within the CloudStack codebase and unless a new version supports a new hypervisor it is not possible to add a host of a hypervisor that is not in part of the enum. It is possible to implement minimal changes in CloudStack to support a new hypervisor plugin that may be developed privately This PR is an initial work on allowing new dynamic hypervisor types (adds a new element to the HypervisorType enum, but allows variable display name for the hypervisor) ##### Proposed Future work: Replace the HypervisorType from a fixed enum to an extensible registry mechanism, registered from the hypervisor plugin #### Feature Specifications - The new hypervisor type is internally named 'Custom' to the CloudStack services (management server and agent services, database records). - A new global setting ‘hypervisor.custom.display.name’ allows administrators to set the display name of the hypervisor type. The display name will be shown in the CloudStack UI and API. - In case the ‘hypervisor.list’ setting contains the display name of the new hypervisor type, the setting value is automatically updated after the ‘hypervisor.custom.display.name’ setting is updated. - The new Custom hypervisor type supports: - Direct downloads (the ability to download templates into primary storage from the hypervisor hosts without using secondary storage) - Local storage (use hypervisor hosts local storage as primary storage) - Template format: RAW format (the templates to be registered on the new hypervisor type must be in RAW format) - The UI is also extended to display the new hypervisor type and the supported features listed above. - The above are the minimal changes for CloudStack to support the new hypervisor type, which can be tested by integrating the plugin codebase with this feature. #### Use cases This PR allows the cloud administrators to test custom hypervisor plugins implementations in CloudStack and easily integrate it into CloudStack as a new hypervisor type ("Custom"), reducing the implementation to only the hypervisor supported specific storage/networking and the hypervisor resource to communicate with the management server. - CloudStack admin should be able to create a zone for the new custom hypervisor and add clusters, hosts into the zone with normal operations - CloudStack users should be able to execute normal VMs/volumes/network/storage operations on VMs/volumes running on the custom hypervisor hosts --- .../java/com/cloud/hypervisor/Hypervisor.java | 18 +++++++++-- .../com/cloud/hypervisor/HypervisorGuru.java | 5 +++ .../user/template/RegisterTemplateCmd.java | 9 ++++-- .../response/HostForMigrationResponse.java | 5 ++- .../cloudstack/api/response/HostResponse.java | 7 ++-- .../HypervisorCapabilitiesResponse.java | 7 ++-- .../api/response/VMSnapshotResponse.java | 7 ++-- .../cloud/storage/GuestOSHypervisorVO.java | 3 +- .../src/test/resources/component.xml | 5 +++ .../src/main/resources/components-example.xml | 1 + .../main/java/com/cloud/api/ApiDBUtils.java | 3 ++ .../java/com/cloud/api/ApiResponseHelper.java | 13 ++++---- .../query/dao/DomainRouterJoinDaoImpl.java | 2 +- .../cloud/api/query/dao/HostJoinDaoImpl.java | 10 ++++-- .../api/query/dao/StoragePoolJoinDaoImpl.java | 4 +-- .../api/query/dao/TemplateJoinDaoImpl.java | 4 +-- .../api/query/dao/UserVmJoinDaoImpl.java | 2 +- .../api/query/dao/VolumeJoinDaoImpl.java | 12 ++++--- .../ConfigurationManagerImpl.java | 18 +++++++++++ .../cloud/hypervisor/HypervisorGuruBase.java | 5 ++- .../discoverer/CustomServerDiscoverer.java | 32 +++++++++++++++++++ .../cloud/resource/ResourceManagerImpl.java | 5 ++- .../cloud/server/ManagementServerImpl.java | 6 ++-- .../template/HypervisorTemplateAdapter.java | 16 ++++++---- .../cloud/template/TemplateManagerImpl.java | 8 +++-- .../java/com/cloud/vm/UserVmManagerImpl.java | 23 ++++++++++++- .../spring-server-discoverer-context.xml | 5 +++ .../com/cloud/vm/UserVmManagerImplTest.java | 29 ++++++++++++++--- ui/src/config/section/infra/hosts.js | 4 ++- ui/src/store/getters.js | 3 +- ui/src/store/modules/user.js | 27 +++++++++++++++- .../views/image/RegisterOrUploadTemplate.vue | 32 +++++++++++++++++-- .../infra/zone/ZoneWizardAddResources.vue | 7 ++-- 33 files changed, 271 insertions(+), 66 deletions(-) create mode 100644 server/src/main/java/com/cloud/hypervisor/discoverer/CustomServerDiscoverer.java diff --git a/api/src/main/java/com/cloud/hypervisor/Hypervisor.java b/api/src/main/java/com/cloud/hypervisor/Hypervisor.java index 429de5774a4..2f0cc736af3 100644 --- a/api/src/main/java/com/cloud/hypervisor/Hypervisor.java +++ b/api/src/main/java/com/cloud/hypervisor/Hypervisor.java @@ -27,7 +27,7 @@ public class Hypervisor { static Map hypervisorTypeMap; static Map supportedImageFormatMap; - public static enum HypervisorType { + public enum HypervisorType { None, //for storage hosts XenServer, KVM, @@ -40,6 +40,7 @@ public class Hypervisor { Ovm, Ovm3, LXC, + Custom, Any; /*If you don't care about the hypervisor type*/ @@ -57,6 +58,7 @@ public class Hypervisor { hypervisorTypeMap.put("lxc", HypervisorType.LXC); hypervisorTypeMap.put("any", HypervisorType.Any); hypervisorTypeMap.put("ovm3", HypervisorType.Ovm3); + hypervisorTypeMap.put("custom", HypervisorType.Custom); supportedImageFormatMap = new HashMap<>(); supportedImageFormatMap.put(HypervisorType.XenServer, ImageFormat.VHD); @@ -68,7 +70,19 @@ public class Hypervisor { public static HypervisorType getType(String hypervisor) { return hypervisor == null ? HypervisorType.None : - hypervisorTypeMap.getOrDefault(hypervisor.toLowerCase(Locale.ROOT), HypervisorType.None); + (hypervisor.toLowerCase(Locale.ROOT).equalsIgnoreCase( + HypervisorGuru.HypervisorCustomDisplayName.value()) ? Custom : + hypervisorTypeMap.getOrDefault(hypervisor.toLowerCase(Locale.ROOT), HypervisorType.None)); + } + + /** + * Returns the display name of a hypervisor type in case the custom hypervisor is used, + * using the 'hypervisor.custom.display.name' setting. Otherwise, returns hypervisor name + */ + public String getHypervisorDisplayName() { + return !Hypervisor.HypervisorType.Custom.equals(this) ? + this.toString() : + HypervisorGuru.HypervisorCustomDisplayName.value(); } /** diff --git a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java index c7dc0bba109..2dfa707b57b 100644 --- a/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java +++ b/api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import org.apache.cloudstack.backup.Backup; +import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.agent.api.Command; import com.cloud.agent.api.to.NicTO; @@ -35,6 +36,10 @@ import com.cloud.vm.VirtualMachineProfile; public interface HypervisorGuru extends Adapter { + ConfigKey HypervisorCustomDisplayName = new ConfigKey<>(String.class, + "hypervisor.custom.display.name", ConfigKey.CATEGORY_ADVANCED, "Custom", + "Display name for custom hypervisor", true, ConfigKey.Scope.Global, null); + HypervisorType getHypervisorType(); /** diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 1a5e851d884..5709e3ed900 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import com.cloud.hypervisor.HypervisorGuru; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -342,9 +343,11 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Parameter zoneids cannot combine all zones (-1) option with other zones"); - if (isDirectDownload() && !getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString())) { - throw new ServerApiException(ApiErrorCode.PARAM_ERROR, - "Parameter directdownload is only allowed for KVM templates"); + String customHypervisor = HypervisorGuru.HypervisorCustomDisplayName.value(); + if (isDirectDownload() && !(getHypervisor().equalsIgnoreCase(Hypervisor.HypervisorType.KVM.toString()) + || getHypervisor().equalsIgnoreCase(customHypervisor))) { + throw new ServerApiException(ApiErrorCode.PARAM_ERROR, String.format("Parameter directdownload " + + "is only allowed for KVM or %s templates", customHypervisor)); } if (!isDeployAsIs() && osTypeId == null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java index 4ed0cdd8d74..41a0fdc4567 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java @@ -24,7 +24,6 @@ import org.apache.cloudstack.api.EntityReference; import com.cloud.host.Host; import com.cloud.host.Status; -import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @@ -84,7 +83,7 @@ public class HostForMigrationResponse extends BaseResponse { @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "the host hypervisor") - private HypervisorType hypervisor; + private String hypervisor; @SerializedName("cpunumber") @Param(description = "the CPU number of the host") @@ -295,7 +294,7 @@ public class HostForMigrationResponse extends BaseResponse { this.version = version; } - public void setHypervisor(HypervisorType hypervisor) { + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index 5d809cf1553..e1f1e5ee241 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -29,7 +29,6 @@ import org.apache.cloudstack.outofbandmanagement.OutOfBandManagement; import com.cloud.host.Host; import com.cloud.host.Status; -import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @@ -89,7 +88,7 @@ public class HostResponse extends BaseResponseWithAnnotations { @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "the host hypervisor") - private HypervisorType hypervisor; + private String hypervisor; @SerializedName("cpusockets") @Param(description = "the number of CPU sockets on the host") @@ -335,7 +334,7 @@ public class HostResponse extends BaseResponseWithAnnotations { this.version = version; } - public void setHypervisor(HypervisorType hypervisor) { + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } @@ -602,7 +601,7 @@ public class HostResponse extends BaseResponseWithAnnotations { return version; } - public HypervisorType getHypervisor() { + public String getHypervisor() { return hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HypervisorCapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HypervisorCapabilitiesResponse.java index b5e5624bbe5..c19397e0c83 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HypervisorCapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HypervisorCapabilitiesResponse.java @@ -20,7 +20,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import org.apache.cloudstack.api.EntityReference; -import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @@ -37,7 +36,7 @@ public class HypervisorCapabilitiesResponse extends BaseResponse { @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "the hypervisor type") - private HypervisorType hypervisor; + private String hypervisor; @SerializedName(ApiConstants.MAX_GUESTS_LIMIT) @Param(description = "the maximum number of guest vms recommended for this hypervisor") @@ -83,11 +82,11 @@ public class HypervisorCapabilitiesResponse extends BaseResponse { this.hypervisorVersion = hypervisorVersion; } - public HypervisorType getHypervisor() { + public String getHypervisor() { return hypervisor; } - public void setHypervisor(HypervisorType hypervisor) { + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VMSnapshotResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VMSnapshotResponse.java index 37670cf6224..9b553ed0744 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VMSnapshotResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VMSnapshotResponse.java @@ -25,7 +25,6 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; -import com.cloud.hypervisor.Hypervisor; import com.cloud.serializer.Param; import com.cloud.vm.snapshot.VMSnapshot; import com.google.gson.annotations.SerializedName; @@ -111,7 +110,7 @@ public class VMSnapshotResponse extends BaseResponseWithTagInformation implement @SerializedName(ApiConstants.HYPERVISOR) @Param(description = "the type of hypervisor on which snapshot is stored") - private Hypervisor.HypervisorType hypervisor; + private String hypervisor; public VMSnapshotResponse() { tags = new LinkedHashSet(); @@ -266,11 +265,11 @@ public class VMSnapshotResponse extends BaseResponseWithTagInformation implement this.tags = tags; } - public Hypervisor.HypervisorType getHypervisor() { + public String getHypervisor() { return hypervisor; } - public void setHypervisor(Hypervisor.HypervisorType hypervisor) { + public void setHypervisor(String hypervisor) { this.hypervisor = hypervisor; } } diff --git a/engine/schema/src/main/java/com/cloud/storage/GuestOSHypervisorVO.java b/engine/schema/src/main/java/com/cloud/storage/GuestOSHypervisorVO.java index 087649b887a..e900d28a864 100644 --- a/engine/schema/src/main/java/com/cloud/storage/GuestOSHypervisorVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/GuestOSHypervisorVO.java @@ -26,6 +26,7 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.db.GenericDao; @Entity @@ -72,7 +73,7 @@ public class GuestOSHypervisorVO implements GuestOSHypervisor { @Override public String getHypervisorType() { - return hypervisorType; + return Hypervisor.HypervisorType.getType(hypervisorType).toString(); } @Override diff --git a/engine/storage/integration-test/src/test/resources/component.xml b/engine/storage/integration-test/src/test/resources/component.xml index aee37145114..d384d546665 100644 --- a/engine/storage/integration-test/src/test/resources/component.xml +++ b/engine/storage/integration-test/src/test/resources/component.xml @@ -121,6 +121,11 @@ + + + + diff --git a/plugins/network-elements/dns-notifier/src/main/resources/components-example.xml b/plugins/network-elements/dns-notifier/src/main/resources/components-example.xml index c53c0b14ef1..76a6cad9b4a 100755 --- a/plugins/network-elements/dns-notifier/src/main/resources/components-example.xml +++ b/plugins/network-elements/dns-notifier/src/main/resources/components-example.xml @@ -111,6 +111,7 @@ under the License. + diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 162b1f38add..16377e13b03 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1283,6 +1283,9 @@ public class ApiDBUtils { // If this check is not passed, the hypervisor type will remain OVM. type = HypervisorType.KVM; break; + } else if (pool.getHypervisor() == HypervisorType.Custom) { + type = HypervisorType.Custom; + break; } } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 568625c856b..8cd80dd9a52 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -38,6 +38,7 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -735,7 +736,7 @@ public class ApiResponseHelper implements ResponseGenerator { if (vm != null) { vmSnapshotResponse.setVirtualMachineId(vm.getUuid()); vmSnapshotResponse.setVirtualMachineName(StringUtils.isEmpty(vm.getDisplayName()) ? vm.getHostName() : vm.getDisplayName()); - vmSnapshotResponse.setHypervisor(vm.getHypervisorType()); + vmSnapshotResponse.setHypervisor(vm.getHypervisorType().getHypervisorDisplayName()); DataCenterVO datacenter = ApiDBUtils.findZoneById(vm.getDataCenterId()); if (datacenter != null) { vmSnapshotResponse.setZoneId(datacenter.getUuid()); @@ -1437,7 +1438,7 @@ public class ApiResponseHelper implements ResponseGenerator { clusterResponse.setZoneId(dc.getUuid()); clusterResponse.setZoneName(dc.getName()); } - clusterResponse.setHypervisorType(cluster.getHypervisorType().toString()); + clusterResponse.setHypervisorType(cluster.getHypervisorType().getHypervisorDisplayName()); clusterResponse.setClusterType(cluster.getClusterType().toString()); clusterResponse.setAllocationState(cluster.getAllocationState().toString()); clusterResponse.setManagedState(cluster.getManagedState().toString()); @@ -1633,7 +1634,7 @@ public class ApiResponseHelper implements ResponseGenerator { vmResponse.setTemplateName(template.getName()); } vmResponse.setCreated(vm.getCreated()); - vmResponse.setHypervisor(vm.getHypervisorType().toString()); + vmResponse.setHypervisor(vm.getHypervisorType().getHypervisorDisplayName()); if (vm.getHostId() != null) { Host host = ApiDBUtils.findHostById(vm.getHostId()); @@ -2792,7 +2793,7 @@ public class ApiResponseHelper implements ResponseGenerator { public HypervisorCapabilitiesResponse createHypervisorCapabilitiesResponse(HypervisorCapabilities hpvCapabilities) { HypervisorCapabilitiesResponse hpvCapabilitiesResponse = new HypervisorCapabilitiesResponse(); hpvCapabilitiesResponse.setId(hpvCapabilities.getUuid()); - hpvCapabilitiesResponse.setHypervisor(hpvCapabilities.getHypervisorType()); + hpvCapabilitiesResponse.setHypervisor(hpvCapabilities.getHypervisorType().getHypervisorDisplayName()); hpvCapabilitiesResponse.setHypervisorVersion(hpvCapabilities.getHypervisorVersion()); hpvCapabilitiesResponse.setIsSecurityGroupEnabled(hpvCapabilities.isSecurityGroupEnabled()); hpvCapabilitiesResponse.setMaxGuestsLimit(hpvCapabilities.getMaxGuestsLimit()); @@ -3690,7 +3691,7 @@ public class ApiResponseHelper implements ResponseGenerator { public GuestOsMappingResponse createGuestOSMappingResponse(GuestOSHypervisor guestOSHypervisor) { GuestOsMappingResponse response = new GuestOsMappingResponse(); response.setId(guestOSHypervisor.getUuid()); - response.setHypervisor(guestOSHypervisor.getHypervisorType()); + response.setHypervisor(Hypervisor.HypervisorType.getType(guestOSHypervisor.getHypervisorType()).getHypervisorDisplayName()); response.setHypervisorVersion(guestOSHypervisor.getHypervisorVersion()); response.setOsNameForHypervisor((guestOSHypervisor.getGuestOsName())); response.setIsUserDefined(Boolean.valueOf(guestOSHypervisor.getIsUserDefined()).toString()); @@ -4940,7 +4941,7 @@ public class ApiResponseHelper implements ResponseGenerator { response.setId(certificate.getUuid()); response.setAlias(certificate.getAlias()); handleCertificateResponse(certificate.getCertificate(), response); - response.setHypervisor(certificate.getHypervisorType().name()); + response.setHypervisor(certificate.getHypervisorType().getHypervisorDisplayName()); response.setObjectName("directdownloadcertificate"); return response; } diff --git a/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java index 83a89622bd2..e3011bc4d66 100644 --- a/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/DomainRouterJoinDaoImpl.java @@ -126,7 +126,7 @@ public class DomainRouterJoinDaoImpl extends GenericDaoBase implements hostResponse.setCpuNumber(host.getCpus()); hostResponse.setZoneId(host.getZoneUuid()); hostResponse.setDisconnectedOn(host.getDisconnectedOn()); - hostResponse.setHypervisor(host.getHypervisorType()); + if (host.getHypervisorType() != null) { + String hypervisorType = host.getHypervisorType().getHypervisorDisplayName(); + hostResponse.setHypervisor(hypervisorType); + } hostResponse.setHostType(host.getType()); hostResponse.setLastPinged(new Date(host.getLastPinged())); Long mshostId = host.getManagementServerId(); @@ -239,7 +242,8 @@ public class HostJoinDaoImpl extends GenericDaoBase implements hostResponse.setUefiCapabilty(new Boolean(false)); } } - if (details.contains(HostDetails.all) && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) { + if (details.contains(HostDetails.all) && (host.getHypervisorType() == Hypervisor.HypervisorType.KVM || + host.getHypervisorType() == Hypervisor.HypervisorType.Custom)) { //only kvm has the requirement to return host details try { hostResponse.setDetails(hostDetails); @@ -303,7 +307,7 @@ public class HostJoinDaoImpl extends GenericDaoBase implements hostResponse.setCpuNumber(host.getCpus()); hostResponse.setZoneId(host.getZoneUuid()); hostResponse.setDisconnectedOn(host.getDisconnectedOn()); - hostResponse.setHypervisor(host.getHypervisorType()); + hostResponse.setHypervisor(host.getHypervisorType().getHypervisorDisplayName()); hostResponse.setHostType(host.getType()); hostResponse.setLastPinged(new Date(host.getLastPinged())); hostResponse.setManagementServerId(host.getManagementServerId()); diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 527cc949ed1..de469d21a11 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -110,7 +110,7 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase[] getConfigKeys() { - return new ConfigKey[] {VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor, VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor }; + return new ConfigKey[] {VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor, + VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor, + HypervisorCustomDisplayName + }; } } diff --git a/server/src/main/java/com/cloud/hypervisor/discoverer/CustomServerDiscoverer.java b/server/src/main/java/com/cloud/hypervisor/discoverer/CustomServerDiscoverer.java new file mode 100644 index 00000000000..534947f092e --- /dev/null +++ b/server/src/main/java/com/cloud/hypervisor/discoverer/CustomServerDiscoverer.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.discoverer; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.kvm.discoverer.LibvirtServerDiscoverer; + +public class CustomServerDiscoverer extends LibvirtServerDiscoverer { + @Override + public Hypervisor.HypervisorType getHypervisorType() { + return Hypervisor.HypervisorType.Custom; + } + + @Override + protected String getPatchPath() { + return "scripts/vm/hypervisor/kvm/"; + } +} diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 150aed65a36..d1697646c39 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -42,6 +42,7 @@ import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; +import com.cloud.hypervisor.HypervisorGuru; import org.apache.cloudstack.alert.AlertService; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -652,7 +653,9 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } - return discoverHostsFull(dcId, podId, clusterId, clusterName, url, username, password, cmd.getHypervisor(), hostTags, cmd.getFullUrlParams(), false); + String hypervisorType = cmd.getHypervisor().equalsIgnoreCase(HypervisorGuru.HypervisorCustomDisplayName.value()) ? + "Custom" : cmd.getHypervisor(); + return discoverHostsFull(dcId, podId, clusterId, clusterName, url, username, password, hypervisorType, hostTags, cmd.getFullUrlParams(), false); } @Override diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index c16dc4eb2f4..5791f4bf789 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -1264,7 +1264,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } if (hypervisorType != null) { - sc.setParameters("hypervisorType", hypervisorType); + String hypervisorStr = (String) hypervisorType; + String hypervisorSearch = HypervisorType.getType(hypervisorStr).toString(); + sc.setParameters("hypervisorType", hypervisorSearch); } if (clusterType != null) { @@ -4470,7 +4472,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } else { final List clustersForZone = _clusterDao.listByZoneId(zoneId); for (final ClusterVO cluster : clustersForZone) { - result.add(cluster.getHypervisorType().toString()); + result.add(cluster.getHypervisorType().getHypervisorDisplayName()); } } diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index f5c19ecff1d..3f45505bcc5 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -148,20 +148,21 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } /** - * Validate on random running KVM host that URL is reachable + * Validate on random running host that URL is reachable * @param url url */ - private Long performDirectDownloadUrlValidation(final String format, final String url, final List zoneIds) { + private Long performDirectDownloadUrlValidation(final String format, final Hypervisor.HypervisorType hypervisor, + final String url, final List zoneIds) { HostVO host = null; if (zoneIds != null && !zoneIds.isEmpty()) { for (Long zoneId : zoneIds) { - host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + host = resourceManager.findOneRandomRunningHostByHypervisor(hypervisor, zoneId); if (host != null) { break; } } } else { - host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, null); + host = resourceManager.findOneRandomRunningHostByHypervisor(hypervisor, null); } if (host == null) { @@ -198,7 +199,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { zoneIds = new ArrayList<>(); zoneIds.add(cmd.getZoneId()); } - Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), url, zoneIds); + Long templateSize = performDirectDownloadUrlValidation(ImageFormat.ISO.getFileExtension(), + Hypervisor.HypervisorType.KVM, url, zoneIds); profile.setSize(templateSize); } profile.setUrl(url); @@ -221,9 +223,11 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { TemplateProfile profile = super.prepare(cmd); String url = profile.getUrl(); UriUtils.validateUrl(cmd.getFormat(), url, cmd.isDirectDownload()); + Hypervisor.HypervisorType hypervisor = Hypervisor.HypervisorType.getType(cmd.getHypervisor()); if (cmd.isDirectDownload()) { DigestHelper.validateChecksumString(cmd.getChecksum()); - Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), url, cmd.getZoneIds()); + Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), + hypervisor, url, cmd.getZoneIds()); profile.setSize(templateSize); } profile.setUrl(url); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 1a360c88edb..790d9163bf0 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -309,8 +309,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (type == HypervisorType.BareMetal) { adapter = AdapterBase.getAdapterByName(_adapters, TemplateAdapterType.BareMetal.getName()); } else { - // see HypervisorTemplateAdapter - adapter = AdapterBase.getAdapterByName(_adapters, TemplateAdapterType.Hypervisor.getName()); + // Get template adapter according to hypervisor + adapter = AdapterBase.getAdapterByName(_adapters, type.name()); + // Otherwise, default to generic hypervisor template adapter + if (adapter == null) { + adapter = AdapterBase.getAdapterByName(_adapters, TemplateAdapterType.Hypervisor.getName()); + } } if (adapter == null) { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 1dd6d6541e9..3ddaf6b58d3 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -650,6 +650,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir HypervisorType.Simulator )); + protected static final List ROOT_DISK_SIZE_OVERRIDE_SUPPORTING_HYPERVISORS = Arrays.asList( + HypervisorType.KVM, + HypervisorType.XenServer, + HypervisorType.VMware, + HypervisorType.Simulator, + HypervisorType.Custom + ); + private static final List HYPERVISORS_THAT_CAN_DO_STORAGE_MIGRATION_ON_NON_USER_VMS = Arrays.asList(HypervisorType.KVM, HypervisorType.VMware); @Override @@ -4338,7 +4346,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir * @throws InvalidParameterValueException if the hypervisor does not support rootdisksize override */ protected void verifyIfHypervisorSupportsRootdiskSizeOverride(HypervisorType hypervisorType) { - if (!(hypervisorType == HypervisorType.KVM || hypervisorType == HypervisorType.XenServer || hypervisorType == HypervisorType.VMware || hypervisorType == HypervisorType.Simulator)) { + if (!ROOT_DISK_SIZE_OVERRIDE_SUPPORTING_HYPERVISORS.contains(hypervisorType)) { throw new InvalidParameterValueException("Hypervisor " + hypervisorType + " does not support rootdisksize override"); } } @@ -5004,6 +5012,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir Answer startAnswer = cmds.getAnswer(StartAnswer.class); String returnedIp = null; String originalIp = null; + String originalVncPassword = profile.getVirtualMachine().getVncPassword(); + String returnedVncPassword = null; if (startAnswer != null) { StartAnswer startAns = (StartAnswer)startAnswer; VirtualMachineTO vmTO = startAns.getVirtualMachine(); @@ -5012,6 +5022,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir returnedIp = nicTO.getIp(); } } + returnedVncPassword = vmTO.getVncPassword(); } List nics = _nicDao.listByVmId(vm.getId()); @@ -5063,6 +5074,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } + updateVncPasswordIfItHasChanged(originalVncPassword, returnedVncPassword, profile); + // get system ip and create static nat rule for the vm try { _rulesMgr.getSystemIpAndEnableStaticNatForVm(profile.getVirtualMachine(), false); @@ -5097,6 +5110,14 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return true; } + protected void updateVncPasswordIfItHasChanged(String originalVncPassword, String returnedVncPassword, VirtualMachineProfile profile) { + if (returnedVncPassword != null && !originalVncPassword.equals(returnedVncPassword)) { + UserVmVO userVm = _vmDao.findById(profile.getId()); + userVm.setVncPassword(returnedVncPassword); + _vmDao.update(userVm.getId(), userVm); + } + } + @Override public void finalizeExpunge(VirtualMachine vm) { } diff --git a/server/src/main/resources/META-INF/cloudstack/server-discoverer/spring-server-discoverer-context.xml b/server/src/main/resources/META-INF/cloudstack/server-discoverer/spring-server-discoverer-context.xml index 3a7e0ffb04a..98abb08ae11 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-discoverer/spring-server-discoverer-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-discoverer/spring-server-discoverer-context.xml @@ -38,6 +38,11 @@ + + + + diff --git a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java index ef001906d98..9c591690bbb 100644 --- a/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java +++ b/server/src/test/java/com/cloud/vm/UserVmManagerImplTest.java @@ -221,6 +221,9 @@ public class UserVmManagerImplTest { @Mock UserDataManager userDataManager; + @Mock + VirtualMachineProfile virtualMachineProfile; + private static final long vmId = 1l; private static final long zoneId = 2L; private static final long accountId = 3L; @@ -248,6 +251,8 @@ public class UserVmManagerImplTest { customParameters.put(VmDetailConstants.ROOT_DISK_SIZE, "123"); lenient().doNothing().when(resourceLimitMgr).incrementResourceCount(anyLong(), any(Resource.ResourceType.class)); lenient().doNothing().when(resourceLimitMgr).decrementResourceCount(anyLong(), any(Resource.ResourceType.class), anyLong()); + + Mockito.when(virtualMachineProfile.getId()).thenReturn(vmId); } @After @@ -561,13 +566,10 @@ public class UserVmManagerImplTest { public void verifyIfHypervisorSupportRootdiskSizeOverrideTest() { Hypervisor.HypervisorType[] hypervisorTypeArray = Hypervisor.HypervisorType.values(); int exceptionCounter = 0; - int expectedExceptionCounter = hypervisorTypeArray.length - 4; + int expectedExceptionCounter = hypervisorTypeArray.length - 5; for(int i = 0; i < hypervisorTypeArray.length; i++) { - if (Hypervisor.HypervisorType.KVM == hypervisorTypeArray[i] - || Hypervisor.HypervisorType.XenServer == hypervisorTypeArray[i] - || Hypervisor.HypervisorType.VMware == hypervisorTypeArray[i] - || Hypervisor.HypervisorType.Simulator == hypervisorTypeArray[i]) { + if (UserVmManagerImpl.ROOT_DISK_SIZE_OVERRIDE_SUPPORTING_HYPERVISORS.contains(hypervisorTypeArray[i])) { userVmManagerImpl.verifyIfHypervisorSupportsRootdiskSizeOverride(hypervisorTypeArray[i]); } else { try { @@ -1041,4 +1043,21 @@ public class UserVmManagerImplTest { Pair pair = mockObjectsForChooseVmMigrationDestinationUsingVolumePoolMapTest(false, destinationHost); Assert.assertEquals(destinationHost, userVmManagerImpl.chooseVmMigrationDestinationUsingVolumePoolMap(pair.first(), pair.second(), null)); } + + @Test + public void testUpdateVncPasswordIfItHasChanged() { + String vncPassword = "12345678"; + userVmManagerImpl.updateVncPasswordIfItHasChanged(vncPassword, vncPassword, virtualMachineProfile); + Mockito.verify(userVmDao, Mockito.never()).update(vmId, userVmVoMock); + } + + @Test + public void testUpdateVncPasswordIfItHasChangedNewPassword() { + String vncPassword = "12345678"; + String newPassword = "87654321"; + Mockito.when(userVmVoMock.getId()).thenReturn(vmId); + userVmManagerImpl.updateVncPasswordIfItHasChanged(vncPassword, newPassword, virtualMachineProfile); + Mockito.verify(userVmDao).findById(vmId); + Mockito.verify(userVmDao).update(vmId, userVmVoMock); + } } diff --git a/ui/src/config/section/infra/hosts.js b/ui/src/config/section/infra/hosts.js index 60f30e3f3e0..22a9afc79cd 100644 --- a/ui/src/config/section/infra/hosts.js +++ b/ui/src/config/section/infra/hosts.js @@ -81,7 +81,9 @@ export default { label: 'label.action.secure.host', message: 'message.action.secure.host', dataView: true, - show: (record) => { return record.hypervisor === 'KVM' }, + show: (record) => { + return record.hypervisor === 'KVM' || record.hypervisor === store.getters.customHypervisorName + }, args: ['hostid'], mapping: { hostid: { diff --git a/ui/src/store/getters.js b/ui/src/store/getters.js index d795e3318c1..e6f02028ab1 100644 --- a/ui/src/store/getters.js +++ b/ui/src/store/getters.js @@ -48,7 +48,8 @@ const getters = { twoFaEnabled: state => state.user.twoFaEnabled, twoFaProvider: state => state.user.twoFaProvider, twoFaIssuer: state => state.user.twoFaIssuer, - loginFlag: state => state.user.loginFlag + loginFlag: state => state.user.loginFlag, + customHypervisorName: state => state.user.customHypervisorName } export default getters diff --git a/ui/src/store/modules/user.js b/ui/src/store/modules/user.js index 051380231a2..91861a87a41 100644 --- a/ui/src/store/modules/user.js +++ b/ui/src/store/modules/user.js @@ -64,7 +64,8 @@ const user = { shutdownTriggered: false, twoFaEnabled: false, twoFaProvider: '', - twoFaIssuer: '' + twoFaIssuer: '', + customHypervisorName: 'Custom' }, mutations: { @@ -151,6 +152,9 @@ const user = { }, SET_LOGIN_FLAG: (state, flag) => { state.loginFlag = flag + }, + SET_CUSTOM_HYPERVISOR_NAME (state, name) { + state.customHypervisorName = name } }, @@ -298,6 +302,15 @@ const user = { commit('SET_CLOUDIAN', cloudian) }).catch(ignored => { }) + + api('listConfigurations', { name: 'hypervisor.custom.display.name' }).then(json => { + if (json.listconfigurationsresponse.configuration !== null) { + const config = json.listconfigurationsresponse.configuration[0] + commit('SET_CUSTOM_HYPERVISOR_NAME', config.value) + } + }).catch(error => { + reject(error) + }) }) }, @@ -391,6 +404,15 @@ const user = { }).catch(error => { reject(error) }) + + api('listConfigurations', { name: 'hypervisor.custom.display.name' }).then(json => { + if (json.listconfigurationsresponse.configuration !== null) { + const config = json.listconfigurationsresponse.configuration[0] + commit('SET_CUSTOM_HYPERVISOR_NAME', config.value) + } + }).catch(error => { + reject(error) + }) }) }, UpdateConfiguration ({ commit }) { @@ -411,6 +433,9 @@ const user = { }, SetLoginFlag ({ commit }, loggedIn) { commit('SET_LOGIN_FLAG', loggedIn) + }, + SetCustomHypervisorName ({ commit }, name) { + commit('SET_CUSTOM_HYPERVISOR_NAME', name) } } } diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index 4ed33afce9a..06ca207a0ab 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -154,7 +154,7 @@ - + @@ -398,6 +398,7 @@ export default { userdatapolicylist: {}, defaultOsId: null, hyperKVMShow: false, + hyperCustomShow: false, hyperXenServerShow: false, hyperVMWShow: false, selectedFormat: '', @@ -408,7 +409,8 @@ export default { allowed: false, allowDirectDownload: false, uploadParams: null, - currentForm: ['plus-outlined', 'PlusOutlined'].includes(this.action.currentAction.icon) ? 'Create' : 'Upload' + currentForm: ['plus-outlined', 'PlusOutlined'].includes(this.action.currentAction.icon) ? 'Create' : 'Upload', + customHypervisorName: 'Custom' } }, beforeCreate () { @@ -456,6 +458,7 @@ export default { }) }, fetchData () { + this.fetchCustomHypervisorName() this.fetchZone() this.fetchOsTypes() this.fetchUserData() @@ -516,6 +519,23 @@ export default { }) }) }, + fetchCustomHypervisorName () { + const params = { + name: 'hypervisor.custom.display.name' + } + this.loading = true + api('listConfigurations', params).then(json => { + if (json.listconfigurationsresponse.configuration !== null) { + const config = json.listconfigurationsresponse.configuration[0] + if (config && config.name === params.name) { + this.customHypervisorName = config.value + store.dispatch('SetCustomHypervisorName', this.customHypervisorName) + } + } + }).finally(() => { + this.loading = false + }) + }, fetchZone () { const params = {} let listZones = [] @@ -781,6 +801,13 @@ export default { description: 'TAR' }) break + case this.customHypervisorName: + this.hyperCustomShow = true + format.push({ + id: 'RAW', + description: 'RAW' + }) + break default: break } @@ -839,6 +866,7 @@ export default { this.hyperXenServerShow = false this.hyperVMWShow = false this.hyperKVMShow = false + this.hyperCustomShow = false this.deployasis = false this.allowDirectDownload = false this.selectedFormat = null diff --git a/ui/src/views/infra/zone/ZoneWizardAddResources.vue b/ui/src/views/infra/zone/ZoneWizardAddResources.vue index 4c919af94ed..24ddd264486 100644 --- a/ui/src/views/infra/zone/ZoneWizardAddResources.vue +++ b/ui/src/views/infra/zone/ZoneWizardAddResources.vue @@ -108,6 +108,7 @@ import { nextTick } from 'vue' import { api } from '@/api' import { mixinDevice } from '@/utils/mixin.js' import StaticInputsForm from '@views/infra/zone/StaticInputsForm' +import store from '@/store' export default { components: { @@ -283,7 +284,7 @@ export default { placeHolder: 'message.error.host.name', required: true, display: { - hypervisor: ['VMware', 'BareMetal', 'Ovm', 'Hyperv', 'KVM', 'XenServer', 'LXC', 'Simulator'] + hypervisor: ['VMware', 'BareMetal', 'Ovm', 'Hyperv', 'KVM', 'XenServer', 'LXC', 'Simulator', store.getters.customHypervisorName] } }, { @@ -292,7 +293,7 @@ export default { placeHolder: 'message.error.host.username', required: true, display: { - hypervisor: ['VMware', 'BareMetal', 'Ovm', 'Hyperv', 'KVM', 'XenServer', 'LXC', 'Simulator'] + hypervisor: ['VMware', 'BareMetal', 'Ovm', 'Hyperv', 'KVM', 'XenServer', 'LXC', 'Simulator', store.getters.customHypervisorName] } }, { @@ -329,7 +330,7 @@ export default { required: true, password: true, display: { - hypervisor: ['VMware', 'BareMetal', 'Ovm', 'Hyperv', 'KVM', 'XenServer', 'LXC', 'Simulator'], + hypervisor: ['VMware', 'BareMetal', 'Ovm', 'Hyperv', 'KVM', 'XenServer', 'LXC', 'Simulator', store.getters.customHypervisorName], authmethod: 'password' } }, From b0ae701ca4e03594914610ece8250966f591ad49 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Thu, 17 Aug 2023 17:37:10 +1000 Subject: [PATCH 06/31] Remove Super-Linter action and move `flake8` to pre-commit (#7861) --- .github/workflows/linter.yml | 19 ------------------- .pre-commit-config.yaml | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 3dcddb2e110..e3b6f34c3be 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -27,25 +27,6 @@ concurrency: cancel-in-progress: true jobs: - build: - permissions: - contents: read # for actions/checkout to fetch code - statuses: write # for github/super-linter to mark status of each linter run - name: Super-Linter Check - runs-on: ubuntu-22.04 - steps: - - name: Checkout Code - uses: actions/checkout@v3 - with: - # Full git history is needed to get a proper list of changed files within `super-linter` - fetch-depth: 0 - - name: SuperLinter - uses: github/super-linter/slim@v4.9.6 - env: - DEFAULT_BRANCH: main - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_ALL_CODEBASE: false - VALIDATE_PYTHON_FLAKE8: true pre-commit: name: Run pre-commit runs-on: ubuntu-22.04 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75014782442..73298a1400d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,3 +41,19 @@ repos: - id: mixed-line-ending files: \.(java|md|py|txt|yaml|yml)$ # - id: trailing-whitespace + - repo: https://github.com/pycqa/flake8 + rev: 6.1.0 + hooks: + - id: flake8 + args: [--config, .github/linters/.flake8] + exclude: > + (?x) + ^agent/bindir/cloud-setup-agent\.in$| + ^client/bindir/cloud-update-xenserver-licenses\.in$| + ^cloud-cli/bindir/cloud-tool$| + ^python/bindir/cloud-grab-dependent-library-versions$| + ^python/bindir/cloud-setup-baremetal$| + ^scripts/vm/hypervisor/xenserver/storagePlugin$| + ^scripts/vm/hypervisor/xenserver/vmopspremium$| + ^setup/bindir/cloud-setup-encryption\.in$| + ^venv/.*$ From c8d6e505393868b18ea8f3fd7dd75732c292998d Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 17 Aug 2023 10:42:42 +0200 Subject: [PATCH 07/31] VMware: add support for 8.0b (8.0.0.2), 8.0c (8.0.0.3) (#7380) * VMware: add support for 8.0b (8.0.0.2) * VMware 8: add new guest os mappings in VirtualMachineGuestOsIdentifier The full list can be found at https://developer.vmware.com/apis/1355/vsphere * VMware: get guest os mappings of parent version * VMware8: remove guest os mappings for 8.0.0.2 * VMware8: fix code smells * vmware: remove annotations in VmwareVmImplementerTest which caused 0.0% code coverage * VMware8: add a unit test case * VMware: add support for 8.0c (8.0.0.3) * VMware8: move to CloudStackVersion.getVMwareParentVersion * VMware: add support for 8.0u1 (8.0.1.0) * Copy engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java from PR 6979 * Copy engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java from PR 6979 * VMware: ignore the last number in VMware versions * VMware: copy guest os mapping from 8.0 to 8.0.1 * VMware: add unit tests in VmwareVmImplementerTest.java * Copy engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java from PR 6979 * VMware8: retry vm poweron if fails due to exception "File system specific implementation of Ioctl[file] failed" This fixes a weird issue on vmware8. When power on a vm, sometimes it fails due to error 2023-04-27 07:04:43,207 ERROR [c.c.h.v.r.VmwareResource] (DirectAgent-442:ctx-cdd42b03 10.0.32.133, job-105/job-106, cmd: StartCommand) (logid:8a24a607) StartCommand failed due to [Exception: java.lang.RuntimeException Message: File system specific implementation of Ioctl[file] failed ]. java.lang.RuntimeException: File system specific implementation of Ioctl[file] failed at com.cloud.hypervisor.vmware.util.VmwareClient.waitForTask(VmwareClient.java:426) at com.cloud.hypervisor.vmware.mo.VirtualMachineMO.powerOn(VirtualMachineMO.java:288) in vmware.log on ESXi host, it shows 2023-04-27T09:20:41.713Z In(05)+ vmx - Power on failure messages: File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of LookupAndOpen[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - File system specific implementation of Ioctl[file] failed 2023-04-27T09:20:41.713Z In(05)+ vmx - Failed to lock the file 2023-04-27T09:20:41.713Z In(05)+ vmx - Cannot open the disk '/vmfs/volumes/7b29c876-ac102328/i-2-167-VM/ROOT-167.vmdk' or one of the snapshot disks it depends on. 2023-04-27T09:20:41.713Z In(05)+ vmx - Module 'Disk' power on failed. 2023-04-27T09:20:41.713Z In(05)+ vmx - Failed to start the virtual machine. There is a KB article for it, but I still do not know why and how to fix it. https://kb.vmware.com/s/article/1004232 * VMware: extract to method powerOnVM * vmware: fix mistake in logs * vmware8: use curl instead of wget to fix test failures Traceback (most recent call last): File "/root/test_internal_lb.py", line 555, in test_01_internallb_roundrobin_1VPC_3VM_HTTP_port80 self.execute_internallb_roundrobin_tests(vpc_offering) File "/root/test_internal_lb.py", line 641, in execute_internallb_roundrobin_tests client_vm, applb.sourceipaddress, max_http_requests) File "/root/test_internal_lb.py", line 497, in run_ssh_test_accross_hosts (e, clienthost.public_ip)) AssertionError: list index out of range: SSH failed for VM with IP Address: 10.0.52.187 and sshClient: DEBUG: {Cmd: /usr/bin/wget -T3 -qO- --user=admin --password=password http://10.1.2.253:8081/admin?stats via Host: 10.0.52.188} {returns: ["/usr/bin/wget: '/usr/lib/libpcre.so.1' is not an ELF file", "/usr/bin/wget: can't load library 'libpcre.so.1'"]} * VMware: correct guest OS names in hypervisor mappings for VMware 8.0 el9 and variants were introduced by https://github.com/apache/cloudstack/pull/7059 they are supported with guest os identifiers since VMware 8.0 see https://vdc-repo.vmware.com/vmwb-repository/dcr-public/c476b64b-c93c-4b21-9d76-be14da0148f9/04ca12ad-59b9-4e1c-8232-fd3d4276e52c/SDK/vsphere-ws/docs/ReferenceGuide/vim.vm.GuestOsDescriptor.GuestOsIdentifier.html * VMware: add Ubuntu 20.04 and 22.04 support for vmware 7.0+ * PR7380: only add guest os mappings for Ubuntu 20.04 * PR7380: Correct RHEL9 guest os names and others for VMware 8.0 * PR7380: correct guest os names on 8.0.0.1 as well * PR7380: remove Windows 12 and Windows Server 2025 which are not released yet --- .../dao/HypervisorCapabilitiesDaoImpl.java | 7 ++ .../storage/dao/GuestOSHypervisorDao.java | 2 + .../storage/dao/GuestOSHypervisorDaoImpl.java | 13 ++ .../java/com/cloud/upgrade/GuestOsMapper.java | 51 +++++++- .../upgrade/dao/Upgrade41800to41810.java | 114 ++++++++++++++++++ .../META-INF/db/schema-41800to41810.sql | 3 + .../com/cloud/upgrade/GuestOsMapperTest.java | 84 +++++++++++++ .../hypervisor/guru/VmwareVmImplementer.java | 16 ++- .../vmware/resource/VmwareResource.java | 21 +++- .../guru/VmwareVmImplementerTest.java | 52 +++++++- test/integration/smoke/test_internal_lb.py | 7 +- .../cloudstack/utils/CloudStackVersion.java | 17 +++ .../utils/CloudStackVersionTest.java | 20 +++ 13 files changed, 393 insertions(+), 14 deletions(-) create mode 100644 engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java diff --git a/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java b/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java index 9a1f29014f9..e8272825213 100644 --- a/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/hypervisor/dao/HypervisorCapabilitiesDaoImpl.java @@ -19,6 +19,7 @@ package com.cloud.hypervisor.dao; import java.util.ArrayList; import java.util.List; +import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -73,6 +74,12 @@ public class HypervisorCapabilitiesDaoImpl extends GenericDaoBase sc = HypervisorTypeAndVersionSearch.create(); sc.setParameters("hypervisorType", hypervisorType); sc.setParameters("hypervisorVersion", hypervisorVersion); + HypervisorCapabilitiesVO result = findOneBy(sc); + if (result != null || !HypervisorType.VMware.equals(hypervisorType) || + CloudStackVersion.getVMwareParentVersion(hypervisorVersion) == null) { + return result; + } + sc.setParameters("hypervisorVersion", CloudStackVersion.getVMwareParentVersion(hypervisorVersion)); return findOneBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java index 17c6b3c959d..47a71433a32 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java @@ -40,4 +40,6 @@ public interface GuestOSHypervisorDao extends GenericDao listHypervisorSupportedVersionsFromMinimumVersion(String hypervisorType, String hypervisorVersion); + + List listByHypervisorTypeAndVersion(String hypervisorType, String hypervisorVersion); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java index ae3ae9aceb8..65f17c29fbc 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java @@ -39,6 +39,7 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase userDefinedMappingSearch; protected final SearchBuilder guestOsNameSearch; protected final SearchBuilder availableHypervisorVersionSearch; + protected final SearchBuilder hypervisorTypeAndVersionSearch; public GuestOSHypervisorDaoImpl() { guestOsSearch = createSearchBuilder(); @@ -73,6 +74,11 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase listByHypervisorTypeAndVersion(String hypervisorType, String hypervisorVersion) { + SearchCriteria sc = hypervisorTypeAndVersionSearch.create(); + sc.setParameters("hypervisor_type", hypervisorType); + sc.setParameters("hypervisor_version", hypervisorVersion); + return listIncludingRemovedBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java b/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java index def77c58f82..739eb32261c 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java @@ -17,6 +17,7 @@ package com.cloud.upgrade; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import java.sql.Connection; @@ -26,6 +27,7 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorMapping; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.storage.GuestOSVO; @@ -94,7 +96,7 @@ public class GuestOsMapper { } } - private boolean addGuestOs(long categoryId, String displayName) { + public boolean addGuestOs(long categoryId, String displayName) { LOG.debug("Adding guest OS with category id: " + categoryId + " and display name: " + displayName); GuestOSVO guestOS = new GuestOSVO(); guestOS.setCategoryId(categoryId); @@ -116,7 +118,7 @@ public class GuestOsMapper { return; } - LOG.debug("Adding guest OS hypervisor mapping - " + mapping.toString()); + LOG.debug("Adding guest OS hypervisor mapping - " + mapping.toString() + ", for guest OS with id - " + guestOsId); GuestOSHypervisorVO guestOsMapping = new GuestOSHypervisorVO(); guestOsMapping.setHypervisorType(mapping.getHypervisorType()); guestOsMapping.setHypervisorVersion(mapping.getHypervisorVersion()); @@ -198,4 +200,49 @@ public class GuestOsMapper { LOG.warn("Invalid Guest OS hypervisor mapping"); return false; } + + /** + * Copies guest OS mappings from src version to dest version for the hypervisor (use this to copy all mappings from older version to newer version during upgrade) + * @return true if copied successfully, else false. + */ + public boolean copyGuestOSHypervisorMappings(HypervisorType hypervisorType, String srcVersion, String destVersion) { + if (hypervisorType == HypervisorType.None || hypervisorType == HypervisorType.Any) { + LOG.warn("Unable to copy, invalid hypervisor"); + return false; + } + + if (StringUtils.isBlank(srcVersion) || StringUtils.isBlank(destVersion)) { + LOG.warn("Unable to copy, invalid hypervisor version details"); + return false; + } + + List guestOSHypervisorMappingsForSrcVersion = guestOSHypervisorDao.listByHypervisorTypeAndVersion(hypervisorType.toString(), srcVersion); + if (CollectionUtils.isEmpty(guestOSHypervisorMappingsForSrcVersion)) { + LOG.warn(String.format("Unable to copy, couldn't find guest OS mappings for hypervisor: %s and src version: %s", hypervisorType.toString(), srcVersion)); + return false; + } + + LOG.debug(String.format("Adding guest OS mappings for hypervisor: %s and version: %s, from version: %s ", hypervisorType.toString(), destVersion, srcVersion)); + for (GuestOSHypervisorVO guestOSHypervisorMapping : guestOSHypervisorMappingsForSrcVersion) { + GuestOSHypervisorMapping mapping = new GuestOSHypervisorMapping(hypervisorType.toString(), destVersion, guestOSHypervisorMapping.getGuestOsName()); + addGuestOsHypervisorMapping(mapping, guestOSHypervisorMapping.getGuestOsId()); + } + return true; + } + + public void updateGuestOsNameInHypervisorMapping(long categoryId, String displayName, GuestOSHypervisorMapping mapping) { + if (!isValidGuestOSHypervisorMapping(mapping)) { + return; + } + + long guestOsId = getGuestOsId(categoryId, displayName); + if (guestOsId == 0) { + LOG.error(String.format("no guest os found for category %d and name %s, skipping mapping it to %s/%s", guestOsId, displayName, mapping.getHypervisorType(), mapping.getHypervisorVersion())); + return; + } + + GuestOSHypervisorVO guestOsMapping = guestOSHypervisorDao.findByOsIdAndHypervisor(guestOsId, mapping.getHypervisorType(), mapping.getHypervisorVersion()); + guestOsMapping.setGuestOsName(mapping.getGuestOsName()); + guestOSHypervisorDao.update(guestOsMapping.getId(), guestOsMapping); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index ab493da8cd8..a2733215060 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -16,6 +16,9 @@ // under the License. package com.cloud.upgrade.dao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.GuestOSHypervisorMapping; +import com.cloud.upgrade.GuestOsMapper; import com.cloud.upgrade.SystemVmTemplateRegistration; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; @@ -58,6 +61,8 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate @Override public void performDataMigration(Connection conn) { fixForeignKeyNames(conn); + updateGuestOsMappings(conn); + copyGuestOsMappingsToVMware80u1(); } @Override @@ -86,6 +91,115 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate } } + private void updateGuestOsMappings(Connection conn) { + LOG.debug("Updating guest OS mappings"); + + GuestOsMapper guestOsMapper = new GuestOsMapper(); + List mappings = new ArrayList<>(); + + LOG.debug("Adding Ubuntu 20.04 support for VMware 6.5+"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.5", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7.1", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7.2", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "6.7.3", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0.1.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0.2.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "7.0.3.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + guestOsMapper.addGuestOsHypervisorMapping(new GuestOSHypervisorMapping("VMware", "8.0", "ubuntu64Guest"), 10, "Ubuntu 20.04 LTS"); + + LOG.debug("Adding Ubuntu 22.04 support for KVM and VMware 6.5+"); + mappings.add(new GuestOSHypervisorMapping("KVM", "default", "Ubuntu 22.04 LTS")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.5", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7.1", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7.2", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "6.7.3", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0.1.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0.2.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "7.0.3.0", "ubuntu64Guest")); + mappings.add(new GuestOSHypervisorMapping("VMware", "8.0", "ubuntu64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(10, "Ubuntu 22.04 LTS", mappings); + mappings.clear(); + + LOG.debug("Correcting guest OS names in hypervisor mappings for VMware 8.0 ad 8.0.0.1"); + final String hypervisorVMware = Hypervisor.HypervisorType.VMware.name(); + final String hypervisorVersionVmware8 = "8.0"; + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "AlmaLinux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "almalinux_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Oracle Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "oracleLinux9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Rocky Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "rockylinux_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "AlmaLinux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "almalinux_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Oracle Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "oracleLinux9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Rocky Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "rockylinux_64Guest")); + + LOG.debug("Correcting guest OS names in hypervisor mappings for Red Hat Enterprise Linux 9"); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0.1.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0.2.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "7.0.3.0", "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "rhel9_64Guest")); + guestOsMapper.updateGuestOsNameInHypervisorMapping(1, "Red Hat Enterprise Linux 9", new GuestOSHypervisorMapping(hypervisorVMware, "8.0.0.1", "rhel9_64Guest")); + + LOG.debug("Adding new guest OS ids in hypervisor mappings for VMware 8.0"); + // Add support for darwin22_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "darwin22_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "macOS 13 (64-bit)", mappings); + mappings.clear(); + + // Add support for darwin23_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "darwin23_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "macOS 14 (64-bit)", mappings); + mappings.clear(); + + // Add support for debian12_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "debian12_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(2, "Debian GNU/Linux 12 (64-bit)", mappings); + mappings.clear(); + + // Add support for debian12Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "debian12Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(2, "Debian GNU/Linux 12 (32-bit)", mappings); + mappings.clear(); + + // Add support for freebsd14_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "freebsd14_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(9, "FreeBSD 14 (64-bit)", mappings); + mappings.clear(); + + // Add support for freebsd14Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "freebsd14Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(9, "FreeBSD 14 (32-bit)", mappings); + mappings.clear(); + + // Add support for other6xLinux64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "other6xLinux64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "Linux 6.x Kernel (64-bit)", mappings); + mappings.clear(); + + // Add support for other6xLinuxGuest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "other6xLinuxGuest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "Linux 6.x Kernel (32-bit)", mappings); + mappings.clear(); + + // Add support for vmkernel8Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "vmkernel8Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(7, "VMware ESXi 8.0", mappings); + mappings.clear(); + + // Add support for windows11_64Guest from VMware 8.0 + mappings.add(new GuestOSHypervisorMapping(hypervisorVMware, hypervisorVersionVmware8, "windows11_64Guest")); + guestOsMapper.addGuestOsAndHypervisorMappings(6, "Windows 11 (64-bit)", mappings); + mappings.clear(); + } + + private void copyGuestOsMappingsToVMware80u1() { + LOG.debug("Copying guest OS mappings from VMware 8.0 to VMware 8.0.1"); + GuestOsMapper guestOsMapper = new GuestOsMapper(); + guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.VMware, "8.0", "8.0.1"); + } + private void fixForeignKeyNames(Connection conn) { //Alter foreign key name for user_vm table from fk_user_data_id to fk_user_vm__user_data_id (if exists) List keys = new ArrayList(); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql index 07ac5c8f166..e07e12871f3 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql @@ -19,6 +19,9 @@ -- Schema upgrade from 4.18.0.0 to 4.18.1.0 --; +-- Add support for VMware 8.0u1 (8.0.1.x) +INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.1', 1024, 0, 59, 64, 1, 1); + -- Update conserve_mode of the default network offering for Tungsten Fabric (this fixes issue #7241) UPDATE `cloud`.`network_offerings` SET conserve_mode = 0 WHERE unique_name ='DefaultTungstenFarbicNetworkOffering'; diff --git a/engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java b/engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java new file mode 100644 index 00000000000..fa6c693fe63 --- /dev/null +++ b/engine/schema/src/test/java/com/cloud/upgrade/GuestOsMapperTest.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.upgrade; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.GuestOSHypervisorVO; +import com.cloud.storage.dao.GuestOSHypervisorDao; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.powermock.modules.junit4.PowerMockRunner; + +import java.util.ArrayList; +import java.util.List; + +@RunWith(PowerMockRunner.class) +public class GuestOsMapperTest { + + @Spy + @InjectMocks + GuestOsMapper guestOsMapper = new GuestOsMapper(); + + @Mock + GuestOSHypervisorDao guestOSHypervisorDao; + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testCopyGuestOSHypervisorMappingsFailures() { + boolean result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.Any, "6.0", "7.0"); + Assert.assertFalse(result); + + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.None, "6.0", "7.0"); + Assert.assertFalse(result); + + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "", "7.0"); + Assert.assertFalse(result); + + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "6.0", ""); + Assert.assertFalse(result); + + Mockito.when(guestOSHypervisorDao.listByHypervisorTypeAndVersion(Mockito.anyString(), Mockito.anyString())).thenReturn(null); + result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "6.0", "7.0"); + Assert.assertFalse(result); + } + + @Test + public void testCopyGuestOSHypervisorMappingsSuccess() { + GuestOSHypervisorVO guestOSHypervisorVO = Mockito.mock(GuestOSHypervisorVO.class); + List guestOSHypervisorVOS = new ArrayList<>(); + guestOSHypervisorVOS.add(guestOSHypervisorVO); + Mockito.when(guestOSHypervisorDao.listByHypervisorTypeAndVersion(Mockito.anyString(), Mockito.anyString())).thenReturn(guestOSHypervisorVOS); + Mockito.when(guestOSHypervisorVO.getGuestOsName()).thenReturn("centos"); + GuestOSHypervisorVO guestOsMapping = Mockito.mock(GuestOSHypervisorVO.class); + Mockito.when(guestOSHypervisorDao.persist(guestOsMapping)).thenReturn(guestOsMapping); + + boolean result = guestOsMapper.copyGuestOSHypervisorMappings(Hypervisor.HypervisorType.XenServer, "6.0", "7.0"); + Assert.assertTrue(result); + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java index 100e3d416a7..990a1875a57 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VmwareVmImplementer.java @@ -47,6 +47,7 @@ import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.NicDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.image.deployasis.DeployAsIsHelper; +import org.apache.cloudstack.utils.CloudStackVersion; import org.apache.commons.lang.BooleanUtils; import org.apache.log4j.Logger; @@ -165,7 +166,7 @@ class VmwareVmImplementer { GuestOSHypervisorVO guestOsMapping = null; if (host != null) { - guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), host.getHypervisorVersion()); + guestOsMapping = getGuestOsMapping(guestOS, host.getHypervisorVersion()); } if (guestOsMapping == null || host == null) { to.setPlatformEmulator(null); @@ -405,4 +406,17 @@ class VmwareVmImplementer { return listForSort.toArray(new NicTO[0]); } + + protected GuestOSHypervisorVO getGuestOsMapping(GuestOSVO guestOS , String hypervisorVersion) { + GuestOSHypervisorVO guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), hypervisorVersion); + if (guestOsMapping == null) { + LOGGER.debug(String.format("Cannot find guest os mappings for guest os \"%s\" on VMware %s", guestOS.getDisplayName(), hypervisorVersion)); + String parentVersion = CloudStackVersion.getVMwareParentVersion(hypervisorVersion); + if (parentVersion != null) { + guestOsMapping = guestOsHypervisorDao.findByOsIdAndHypervisor(guestOS.getId(), Hypervisor.HypervisorType.VMware.toString(), parentVersion); + LOGGER.debug(String.format("Found guest os mappings for guest os \"%s\" on VMware %s: %s", guestOS.getDisplayName(), parentVersion, guestOsMapping)); + } + } + return guestOsMapping; + } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 8b833b8221a..01d6f6816e0 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -2627,7 +2627,9 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes // // Power-on VM // - if (!vmMo.powerOn()) { + if (powerOnVM(vmMo, vmInternalCSName, vmNameOnVcenter)) { + s_logger.debug(String.format("VM %s has been started successfully with hostname %s.", vmInternalCSName, vmNameOnVcenter)); + } else { throw new Exception("Failed to start VM. vmName: " + vmInternalCSName + " with hostname " + vmNameOnVcenter); } @@ -2699,6 +2701,23 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } } + private boolean powerOnVM(final VirtualMachineMO vmMo, final String vmInternalCSName, final String vmNameOnVcenter) throws Exception { + int retry = 20; + while (retry-- > 0) { + try { + return vmMo.powerOn(); + } catch (Exception e) { + s_logger.info(String.format("Got exception while power on VM %s with hostname %s", vmInternalCSName, vmNameOnVcenter), e); + if (e.getMessage() != null && e.getMessage().contains("File system specific implementation of Ioctl[file] failed")) { + s_logger.debug(String.format("Failed to power on VM %s with hostname %s. Retrying", vmInternalCSName, vmNameOnVcenter)); + } else { + throw e; + } + } + } + return false; + } + private boolean multipleIsosAtached(DiskTO[] sortedDisks) { return Arrays.stream(sortedDisks).filter(disk -> disk.getType() == Volume.Type.ISO).count() > 1; } diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java index 309eb6d35c4..f1647808c94 100755 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/guru/VmwareVmImplementerTest.java @@ -23,7 +23,7 @@ import static org.mockito.Mockito.verify; import java.util.HashMap; import java.util.Map; -import org.apache.cloudstack.framework.config.ConfigKey; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,17 +32,15 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; -import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.support.AnnotationConfigContextLoader; import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.storage.GuestOSHypervisorVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.dao.GuestOSHypervisorDao; import com.cloud.vm.VmDetailConstants; @RunWith(PowerMockRunner.class) -@PrepareForTest({ConfigKey.class, VmwareVmImplementer.class}) -@ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class VmwareVmImplementerTest { @Spy @@ -52,6 +50,9 @@ public class VmwareVmImplementerTest { @Mock VirtualMachineTO vmTO; + @Mock + GuestOSHypervisorDao guestOsHypervisorDao; + private Map vmDetails = new HashMap(); @Before @@ -145,4 +146,43 @@ public class VmwareVmImplementerTest { executeAndVerifyTest(false, false, "false", false); } + @Test + public void testGetGuestOsMapping1() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + GuestOSHypervisorVO guestOsMapping = Mockito.mock(GuestOSHypervisorVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1.0")).thenReturn(guestOsMapping); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0.1.0"); + Assert.assertEquals(guestOsMapping, result); + } + + @Test + public void testGetGuestOsMapping2() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + GuestOSHypervisorVO guestOsMapping = Mockito.mock(GuestOSHypervisorVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1.0")).thenReturn(null); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1")).thenReturn(guestOsMapping); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0.1.0"); + Assert.assertEquals(guestOsMapping, result); + } + + @Test + public void testGetGuestOsMapping3() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1.0")).thenReturn(null); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0.1")).thenReturn(null); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0.1.0"); + Assert.assertNull(result); + } + + @Test + public void testGetGuestOsMapping4() { + GuestOSVO guestOs = Mockito.mock(GuestOSVO.class); + Mockito.when(guestOs.getId()).thenReturn(200L); + Mockito.when(guestOsHypervisorDao.findByOsIdAndHypervisor(200L, "VMware", "8.0")).thenReturn(null); + GuestOSHypervisorVO result = implementer.getGuestOsMapping(guestOs, "8.0"); + Assert.assertNull(result); + } } diff --git a/test/integration/smoke/test_internal_lb.py b/test/integration/smoke/test_internal_lb.py index 5864f7321bb..8dc341f4d14 100644 --- a/test/integration/smoke/test_internal_lb.py +++ b/test/integration/smoke/test_internal_lb.py @@ -485,7 +485,7 @@ class TestInternalLb(cloudstackTestCase): try: for x in range(0, max_requests): - cmd_test_http = "/usr/bin/wget -T2 -qO- http://" + \ + cmd_test_http = "curl --connect-timeout 3 -L http://" + \ lb_address + "/ 2>/dev/null" # self.debug( "SSH into VM public address: %s and port: %s" # %(.public_ip, vm.public_port)) @@ -677,9 +677,8 @@ class TestInternalLb(cloudstackTestCase): url = "http://" + stats_ip + ":" + \ settings["stats_port"] + settings["stats_uri"] - get_contents = "/usr/bin/wget -T3 -qO- --user=" + \ - settings["username"] + " --password=" + \ - settings["password"] + " " + url + get_contents = "curl --connect-timeout 3 -L --user %s:%s %s" \ + % (settings["username"], settings["password"], url) try: self.logger.debug( "Trying to connect to the haproxy stats url %s" % url) diff --git a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java index 91c5a9758ea..e29bd9c4e17 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/CloudStackVersion.java @@ -272,4 +272,21 @@ public final class CloudStackVersion implements Comparable { public String toString() { return Joiner.on(".").join(asList()); } + + /** + * Get the parent version of VMware hypervisor version + * @since 4.18.1.0 + */ + public static String getVMwareParentVersion(String hypervisorVersion) { + try { + CloudStackVersion version = CloudStackVersion.parse(hypervisorVersion); + String parentVersion = String.format("%s.%s", version.getMajorRelease(), version.getMinorRelease()); + if (version.getPatchRelease() != 0) { + parentVersion = String.format("%s.%s", parentVersion, version.getPatchRelease()); + } + return parentVersion; + } catch (Exception ex) { + return null; + } + } } diff --git a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java index eb7a76a2c0f..dabaf9bc97d 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/CloudStackVersionTest.java @@ -21,6 +21,7 @@ package org.apache.cloudstack.utils; import com.google.common.testing.EqualsTester; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -204,4 +205,23 @@ public final class CloudStackVersionTest { assertEquals(expected, CloudStackVersion.trimRouterVersion(value)); } + + private void verifyGetVMwareParentVersion(String hypervisorVersion, String expectedParentVersion) { + if (expectedParentVersion == null) { + Assert.assertNull(CloudStackVersion.getVMwareParentVersion(hypervisorVersion)); + } else { + Assert.assertEquals(CloudStackVersion.getVMwareParentVersion(hypervisorVersion), expectedParentVersion); + } + } + @Test + public void testGetParentVersion() { + verifyGetVMwareParentVersion(null, null); + verifyGetVMwareParentVersion("6.5", null); + verifyGetVMwareParentVersion("6.7.3", "6.7.3"); + verifyGetVMwareParentVersion("7.0.3.0", "7.0.3"); + verifyGetVMwareParentVersion("8.0", null); + verifyGetVMwareParentVersion("8.0.0", "8.0"); + verifyGetVMwareParentVersion("8.0.0.2", "8.0"); + verifyGetVMwareParentVersion("8.0.1.0", "8.0.1"); + } } From d8a5c8906028314c37f3dd47801a5efda025454f Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Thu, 17 Aug 2023 11:04:14 +0200 Subject: [PATCH 08/31] server: get only Ready ISO to mount (#7848) --- .../src/main/java/com/cloud/template/TemplateManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 1a360c88edb..d1fd204c024 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -1215,7 +1215,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, tmplt = _tmplFactory.getReadyBypassedTemplateOnPrimaryStore(isoId, poolId, hostId); bypassed = true; } else { - tmplt = _tmplFactory.getTemplate(isoId, DataStoreRole.Image, dcId); + tmplt = _tmplFactory.getReadyTemplateOnImageStore(isoId, dcId); } if (tmplt == null || tmplt.getFormat() != ImageFormat.ISO) { From 7049a6058d2fedf96764ef4c709dde23a5431de1 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 17 Aug 2023 10:22:34 -0300 Subject: [PATCH 09/31] UI: Add domain and account fields to Register/Upload Template/ISO view (#7872) * UI: Add missing domain and account parameters to register template * Only display the domain if listDomains API is accessible * Add fields to Register ISO view --- ui/src/views/image/RegisterOrUploadIso.vue | 89 ++++++++++++++++++- .../views/image/RegisterOrUploadTemplate.vue | 86 ++++++++++++++++++ 2 files changed, 174 insertions(+), 1 deletion(-) diff --git a/ui/src/views/image/RegisterOrUploadIso.vue b/ui/src/views/image/RegisterOrUploadIso.vue index c56dcd8aa7b..6c34ec8972a 100644 --- a/ui/src/views/image/RegisterOrUploadIso.vue +++ b/ui/src/views/image/RegisterOrUploadIso.vue @@ -97,6 +97,48 @@ + + + + + + + + {{ opt.path || opt.name || opt.description }} + + + + + + + + + {{ acc.name }} + + + + @@ -229,7 +271,12 @@ export default { allowed: false, uploadParams: null, uploadPercentage: 0, - currentForm: ['plus-outlined', 'PlusOutlined'].includes(this.action.currentAction.icon) ? 'Create' : 'Upload' + currentForm: ['plus-outlined', 'PlusOutlined'].includes(this.action.currentAction.icon) ? 'Create' : 'Upload', + domains: [], + accounts: [], + domainLoading: false, + domainid: null, + account: null } }, beforeCreate () { @@ -270,6 +317,9 @@ export default { this.fetchOsType() this.fetchUserData() this.fetchUserdataPolicy() + if ('listDomains' in this.$store.getters.apis) { + this.fetchDomains() + } }, fetchZoneData () { const params = {} @@ -478,6 +528,43 @@ export default { }).finally(() => { this.loading = false }) + }, + fetchDomains () { + const params = {} + params.listAll = true + params.showicon = true + params.details = 'min' + this.domainLoading = true + api('listDomains', params).then(json => { + this.domains = json.listdomainsresponse.domain + }).finally(() => { + this.domainLoading = false + this.handleDomainChange(null) + }) + }, + handleDomainChange (domain) { + this.domainid = domain + this.form.account = null + this.account = null + if ('listAccounts' in this.$store.getters.apis) { + this.fetchAccounts() + } + }, + fetchAccounts () { + api('listAccounts', { + domainid: this.domainid + }).then(response => { + this.accounts = response.listaccountsresponse.account || [] + }).catch(error => { + this.$notifyError(error) + }) + }, + handleAccountChange (acc) { + if (acc) { + this.account = acc.name + } else { + this.account = acc + } } } } diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index 06ca207a0ab..f9578d4714d 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -117,6 +117,47 @@ + + + + + + + + {{ opt.path || opt.name || opt.description }} + + + + + + + + + {{ acc.name }} + + + @@ -410,6 +451,11 @@ export default { allowDirectDownload: false, uploadParams: null, currentForm: ['plus-outlined', 'PlusOutlined'].includes(this.action.currentAction.icon) ? 'Create' : 'Upload', + domains: [], + accounts: [], + domainLoading: false, + domainid: null, + account: null, customHypervisorName: 'Custom' } }, @@ -463,6 +509,9 @@ export default { this.fetchOsTypes() this.fetchUserData() this.fetchUserdataPolicy() + if ('listDomains' in this.$store.getters.apis) { + this.fetchDomains() + } if (Object.prototype.hasOwnProperty.call(store.getters.apis, 'listConfigurations')) { if (this.allowed && this.hyperXenServerShow) { this.fetchXenServerProvider() @@ -1026,6 +1075,43 @@ export default { arrSelectReset.forEach(name => { this.form[name] = undefined }) + }, + fetchDomains () { + const params = {} + params.listAll = true + params.showicon = true + params.details = 'min' + this.domainLoading = true + api('listDomains', params).then(json => { + this.domains = json.listdomainsresponse.domain + }).finally(() => { + this.domainLoading = false + this.handleDomainChange(null) + }) + }, + handleDomainChange (domain) { + this.domainid = domain + this.form.account = null + this.account = null + if ('listAccounts' in this.$store.getters.apis) { + this.fetchAccounts() + } + }, + fetchAccounts () { + api('listAccounts', { + domainid: this.domainid + }).then(response => { + this.accounts = response.listaccountsresponse.account || [] + }).catch(error => { + this.$notifyError(error) + }) + }, + handleAccountChange (acc) { + if (acc) { + this.account = acc.name + } else { + this.account = acc + } } } } From 87d6c4fbd5f386b8646377cacbb25d30d07284af Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 18 Aug 2023 09:20:00 +0200 Subject: [PATCH 10/31] packaging: install tzdata-java or timezone-java (#7875) According to https://pkgs.org/download/tzdata-java the package name is tzdata-java on some OSes, but timezone-java on some other OSes (e.g openSUSE Leap 15.4) --- packaging/centos8/cloud.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/centos8/cloud.spec b/packaging/centos8/cloud.spec index fdaf155ed73..8d567f217ce 100644 --- a/packaging/centos8/cloud.spec +++ b/packaging/centos8/cloud.spec @@ -53,7 +53,7 @@ intelligent IaaS cloud implementation. %package management Summary: CloudStack management server UI Requires: java-11-openjdk -Requires: tzdata-java +Requires: (tzdata-java or timezone-java) Requires: python3 Requires: bash Requires: gawk From 5b339673109400ff949b91c1c74c709b72f91178 Mon Sep 17 00:00:00 2001 From: sato03 Date: Fri, 18 Aug 2023 04:33:05 -0300 Subject: [PATCH 11/31] Fix role escalation prevention (#7853) Co-authored-by: Henrique Sato --- .../acl/StaticRoleBasedAPIAccessChecker.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/acl/static-role-based/src/main/java/org/apache/cloudstack/acl/StaticRoleBasedAPIAccessChecker.java b/plugins/acl/static-role-based/src/main/java/org/apache/cloudstack/acl/StaticRoleBasedAPIAccessChecker.java index 27f8305f579..7d12178f0f3 100644 --- a/plugins/acl/static-role-based/src/main/java/org/apache/cloudstack/acl/StaticRoleBasedAPIAccessChecker.java +++ b/plugins/acl/static-role-based/src/main/java/org/apache/cloudstack/acl/StaticRoleBasedAPIAccessChecker.java @@ -76,12 +76,12 @@ public class StaticRoleBasedAPIAccessChecker extends AdapterBase implements APIA if (roleService.isEnabled()) { LOGGER.debug("RoleService is enabled. We will use it instead of StaticRoleBasedAPIAccessChecker."); } - return roleService.isEnabled(); + return !roleService.isEnabled(); } @Override public List getApisAllowedToUser(Role role, User user, List apiNames) throws PermissionDeniedException { - if (isEnabled()) { + if (!isEnabled()) { return apiNames; } @@ -93,7 +93,7 @@ public class StaticRoleBasedAPIAccessChecker extends AdapterBase implements APIA @Override public boolean checkAccess(User user, String commandName) throws PermissionDeniedException { - if (isEnabled()) { + if (!isEnabled()) { return true; } @@ -107,6 +107,10 @@ public class StaticRoleBasedAPIAccessChecker extends AdapterBase implements APIA @Override public boolean checkAccess(Account account, String commandName) { + if (!isEnabled()) { + return true; + } + RoleType roleType = accountService.getRoleType(account); if (isApiAllowed(commandName, roleType)) { return true; From add64bd7e69eca52f5a3e10748c247ea50afcc14 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Fri, 18 Aug 2023 10:49:16 +0200 Subject: [PATCH 12/31] security group: replace vm.getPrivateMacAddress() with nic.getMacAddress() (#7293) --- .../com/cloud/network/security/SecurityGroupManagerImpl.java | 2 +- .../com/cloud/network/security/SecurityGroupManagerImpl2.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java index abbbd4965a4..b423ce78fa8 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl.java @@ -1071,7 +1071,7 @@ public class SecurityGroupManagerImpl extends ManagerBase implements SecurityGro } else { return; } - SecurityGroupRulesCmd cmd = generateRulesetCmd(vm.getInstanceName(), nic.getIPv4Address(), nic.getIPv6Address(), vm.getPrivateMacAddress(), vm.getId(), + SecurityGroupRulesCmd cmd = generateRulesetCmd(vm.getInstanceName(), nic.getIPv4Address(), nic.getIPv6Address(), nic.getMacAddress(), vm.getId(), generateRulesetSignature(ingressRules, egressRules), seqnum, ingressRules, egressRules, nicSecIps); Commands cmds = new Commands(cmd); try { diff --git a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java index eadcc5f1e7d..b75c39560cf 100644 --- a/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java +++ b/server/src/main/java/com/cloud/network/security/SecurityGroupManagerImpl2.java @@ -188,7 +188,7 @@ public class SecurityGroupManagerImpl2 extends SecurityGroupManagerImpl { return; } SecurityGroupRulesCmd cmd = - generateRulesetCmd(vm.getInstanceName(), nic.getIPv4Address(), nic.getIPv6Address(), vm.getPrivateMacAddress(), vm.getId(), null, work.getLogsequenceNumber(), + generateRulesetCmd(vm.getInstanceName(), nic.getIPv4Address(), nic.getIPv6Address(), nic.getMacAddress(), vm.getId(), null, work.getLogsequenceNumber(), ingressRules, egressRules, nicSecIps); cmd.setMsId(_serverId); if (s_logger.isDebugEnabled()) { From 1065e9046b15360713a36c364fc25fc77fe362d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jandre?= <48719461+JoaoJandre@users.noreply.github.com> Date: Fri, 18 Aug 2023 05:51:36 -0300 Subject: [PATCH 13/31] Fix backup dates (#6473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: João Paraquetti Co-authored-by: dahn --- .../api/response/BackupResponse.java | 10 +- .../response/BackupRestorePointResponse.java | 10 +- .../org/apache/cloudstack/backup/Backup.java | 14 ++- .../upgrade/dao/Upgrade41810to41900.java | 111 ++++++++++++++++++ .../apache/cloudstack/backup/BackupVO.java | 12 +- .../backup/DummyBackupProvider.java | 2 +- .../backup/NetworkerBackupProvider.java | 11 +- .../backup/networker/NetworkerClient.java | 10 +- .../cloudstack/backup/veeam/VeeamClient.java | 10 +- ui/src/components/view/ListResourceTable.vue | 5 + 10 files changed, 171 insertions(+), 24 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java index d0c8e588450..63419680fea 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupResponse.java @@ -25,6 +25,8 @@ import org.apache.cloudstack.backup.Backup; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import java.util.Date; + @EntityReference(value = Backup.class) public class BackupResponse extends BaseResponse { @@ -50,7 +52,7 @@ public class BackupResponse extends BaseResponse { @SerializedName(ApiConstants.CREATED) @Param(description = "backup date") - private String date; + private Date date; @SerializedName(ApiConstants.SIZE) @Param(description = "backup size in bytes") @@ -140,11 +142,11 @@ public class BackupResponse extends BaseResponse { this.type = type; } - public String getDate() { - return date; + public Date getDate() { + return this.date; } - public void setDate(String date) { + public void setDate(Date date) { this.date = date; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRestorePointResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRestorePointResponse.java index afb3e9ffc5f..22bb099b1b0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupRestorePointResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRestorePointResponse.java @@ -25,6 +25,8 @@ import org.apache.cloudstack.backup.Backup; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import java.util.Date; + @EntityReference(value = Backup.RestorePoint.class) public class BackupRestorePointResponse extends BaseResponse { @@ -34,7 +36,7 @@ public class BackupRestorePointResponse extends BaseResponse { @SerializedName(ApiConstants.CREATED) @Param(description = "created time") - private String created; + private Date created; @SerializedName(ApiConstants.TYPE) @Param(description = "restore point type") @@ -48,11 +50,11 @@ public class BackupRestorePointResponse extends BaseResponse { this.id = id; } - public String getCreated() { - return created; + public Date getCreated() { + return this.created; } - public void setCreated(String created) { + public void setCreated(Date created) { this.created = created; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 75c7ab4cca5..f369367957d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.backup; +import java.util.Date; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; @@ -58,10 +60,10 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { class RestorePoint { private String id; - private String created; + private Date created; private String type; - public RestorePoint(String id, String created, String type) { + public RestorePoint(String id, Date created, String type) { this.id = id; this.created = created; this.type = type; @@ -75,11 +77,11 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { this.id = id; } - public String getCreated() { - return created; + public Date getCreated() { + return this.created; } - public void setCreated(String created) { + public void setCreated(Date created) { this.created = created; } @@ -134,7 +136,7 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { long getVmId(); String getExternalId(); String getType(); - String getDate(); + Date getDate(); Backup.Status getStatus(); Long getSize(); Long getProtectedSize(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java index c9ac468dc53..57f87d1f2cc 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41810to41900.java @@ -17,16 +17,27 @@ package com.cloud.upgrade.dao; import com.cloud.upgrade.SystemVmTemplateRegistration; +import com.cloud.utils.DateUtil; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; import java.io.InputStream; import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; public class Upgrade41810to41900 implements DbUpgrade, DbUpgradeSystemVmTemplate { final static Logger LOG = Logger.getLogger(Upgrade41810to41900.class); private SystemVmTemplateRegistration systemVmTemplateRegistration; + private final SimpleDateFormat[] formats = { + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"), new SimpleDateFormat("MM/dd/yyyy HH:mm:ss"), new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"), + new SimpleDateFormat("EEE MMM dd HH:mm:ss z yyyy")}; + @Override public String[] getUpgradableVersionRange() { return new String[] {"4.18.1.0", "4.19.0.0"}; @@ -55,6 +66,7 @@ public class Upgrade41810to41900 implements DbUpgrade, DbUpgradeSystemVmTemplate @Override public void performDataMigration(Connection conn) { + migrateBackupDates(conn); } @Override @@ -82,4 +94,103 @@ public class Upgrade41810to41900 implements DbUpgrade, DbUpgradeSystemVmTemplate throw new CloudRuntimeException("Failed to find / register SystemVM template(s)"); } } + + public void migrateBackupDates(Connection conn) { + LOG.info("Trying to convert backups' date column from varchar(255) to datetime type."); + + modifyDateColumnNameAndCreateNewOne(conn); + fetchDatesAndMigrateToNewColumn(conn); + dropOldColumn(conn); + + LOG.info("Finished converting backups' date column from varchar(255) to datetime."); + } + + private void modifyDateColumnNameAndCreateNewOne(Connection conn) { + String alterColumnName = "ALTER TABLE `cloud`.`backups` CHANGE COLUMN `date` `old_date` varchar(255);"; + try (PreparedStatement pstmt = conn.prepareStatement(alterColumnName)) { + pstmt.execute(); + } catch (SQLException e) { + String message = String.format("Unable to alter backups' date column name due to [%s].", e.getMessage()); + LOG.error(message, e); + throw new CloudRuntimeException(message, e); + } + + String createNewColumn = "ALTER TABLE `cloud`.`backups` ADD COLUMN `date` DATETIME;"; + try (PreparedStatement pstmt = conn.prepareStatement(createNewColumn)) { + pstmt.execute(); + } catch (SQLException e) { + String message = String.format("Unable to crate new backups' column date due to [%s].", e.getMessage()); + LOG.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + private void fetchDatesAndMigrateToNewColumn(Connection conn) { + String selectBackupDates = "SELECT `id`, `old_date` FROM `cloud`.`backups` WHERE 1;"; + String date; + java.sql.Date reformatedDate; + + try (PreparedStatement pstmt = conn.prepareStatement(selectBackupDates)) { + try (ResultSet result = pstmt.executeQuery()) { + while (result.next()) { + date = result.getString("old_date"); + reformatedDate = tryToTransformStringToDate(date); + updateBackupDate(conn, result.getLong("id"), reformatedDate); + } + } + } catch (SQLException e) { + String message = String.format("Unable to retrieve backup dates due to [%s].", e.getMessage()); + LOG.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + private java.sql.Date tryToTransformStringToDate(String date) { + Date parsedDate = null; + try { + parsedDate = DateUtil.parseTZDateString(date); + } catch (ParseException e) { + for (SimpleDateFormat sdf: formats) { + try { + parsedDate = sdf.parse(date); + } catch (ParseException ex) { + continue; + } + break; + } + } + if (parsedDate == null) { + String msg = String.format("Unable to parse date [%s]. Will change backup date to null.", date); + LOG.error(msg); + return null; + } + + return new java.sql.Date(parsedDate.getTime()); + } + + private void updateBackupDate(Connection conn, long id, java.sql.Date date) { + String updateBackupDate = "UPDATE `cloud`.`backups` SET `date` = ? WHERE `id` = ?;"; + try (PreparedStatement pstmt = conn.prepareStatement(updateBackupDate)) { + pstmt.setDate(1, date); + pstmt.setLong(2, id); + + pstmt.executeUpdate(); + } catch (SQLException e) { + String message = String.format("Unable to update backup date with id [%s] to date [%s] due to [%s].", id, date, e.getMessage()); + LOG.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + + private void dropOldColumn(Connection conn) { + String dropOldColumn = "ALTER TABLE `cloud`.`backups` DROP COLUMN `old_date`;"; + try (PreparedStatement pstmt = conn.prepareStatement(dropOldColumn)) { + pstmt.execute(); + } catch (SQLException e) { + String message = String.format("Unable to drop old_date column due to [%s].", e.getMessage()); + LOG.error(message, e); + throw new CloudRuntimeException(message, e); + } + } + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index dc47fcb6bb3..e5582609d68 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.backup; +import java.util.Date; import java.util.UUID; import javax.persistence.Column; @@ -27,6 +28,8 @@ import javax.persistence.GeneratedValue; import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; @Entity @Table(name = "backups") @@ -49,7 +52,8 @@ public class BackupVO implements Backup { private String backupType; @Column(name = "date") - private String date; + @Temporal(value = TemporalType.DATE) + private Date date; @Column(name = "size") private Long size; @@ -114,11 +118,11 @@ public class BackupVO implements Backup { } @Override - public String getDate() { - return date; + public Date getDate() { + return this.date; } - public void setDate(String date) { + public void setDate(Date date) { this.date = date; } diff --git a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java index 0297ce82f5e..fabc9821fd3 100644 --- a/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java +++ b/plugins/backup/dummy/src/main/java/org/apache/cloudstack/backup/DummyBackupProvider.java @@ -117,7 +117,7 @@ public class DummyBackupProvider extends AdapterBase implements BackupProvider { backup.setVmId(vm.getId()); backup.setExternalId("dummy-external-id"); backup.setType("FULL"); - backup.setDate(new Date().toString()); + backup.setDate(new Date()); backup.setSize(1024L); backup.setProtectedSize(1024000L); backup.setStatus(Backup.Status.BackedUp); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java index 37051f9b5ee..9703203108a 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/NetworkerBackupProvider.java @@ -51,6 +51,8 @@ import javax.inject.Inject; import java.net.URISyntaxException; import java.security.KeyManagementException; import java.security.NoSuchAlgorithmException; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -605,7 +607,14 @@ public class NetworkerBackupProvider extends AdapterBase implements BackupProvid strayBackup.setVmId(vm.getId()); strayBackup.setExternalId(strayNetworkerBackup.getId()); strayBackup.setType(strayNetworkerBackup.getType()); - strayBackup.setDate(strayNetworkerBackup.getSaveTime()); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + try { + strayBackup.setDate(formatterDateTime.parse(strayNetworkerBackup.getSaveTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", strayNetworkerBackup.getSaveTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); + } strayBackup.setStatus(Backup.Status.BackedUp); for ( Backup.VolumeInfo thisVMVol : vm.getBackupVolumeList()) { vmBackupSize += (thisVMVol.getSize() / 1024L /1024L); diff --git a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java index 939543df387..8bb89b635e9 100644 --- a/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java +++ b/plugins/backup/networker/src/main/java/org/apache/cloudstack/backup/networker/NetworkerClient.java @@ -211,7 +211,7 @@ public class NetworkerClient { SimpleDateFormat formatterDate = new SimpleDateFormat("yyyy-MM-dd"); SimpleDateFormat formatterTime = new SimpleDateFormat("HH:mm:ss"); - SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyy-MM-dd'T'HH:mm:ss"); + SimpleDateFormat formatterDateTime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); String startDate = formatterDate.format(backupJobStart); String startTime = formatterTime.format(backupJobStart); @@ -252,7 +252,13 @@ public class NetworkerClient { backup.setVmId(vm.getId()); backup.setExternalId(networkerLatestBackup.getId()); backup.setType(networkerLatestBackup.getType()); - backup.setDate(networkerLatestBackup.getCreationTime()); + try { + backup.setDate(formatterDateTime.parse(networkerLatestBackup.getCreationTime())); + } catch (ParseException e) { + String msg = String.format("Unable to parse date [%s].", networkerLatestBackup.getCreationTime()); + LOG.error(msg, e); + throw new CloudRuntimeException(msg, e); + } backup.setSize(networkerLatestBackup.getSize().getValue()); backup.setProtectedSize(networkerLatestBackup.getSize().getValue()); backup.setStatus(org.apache.cloudstack.backup.Backup.Status.BackedUp); diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 40fbe97028a..1438dca4838 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -34,6 +34,8 @@ import java.util.List; import java.util.Map; import java.util.StringJoiner; import java.util.UUID; +import java.util.Date; +import java.util.Calendar; import javax.net.ssl.SSLContext; import javax.net.ssl.X509TrustManager; @@ -660,7 +662,7 @@ public class VeeamClient { private Backup.RestorePoint getRestorePointFromBlock(String[] parts) { LOG.debug(String.format("Processing block of restore points: [%s].", StringUtils.join(parts, ", "))); String id = null; - String created = null; + Date created = null; String type = null; for (String part : parts) { if (part.matches("Id(\\s)+:(.)*")) { @@ -668,7 +670,11 @@ public class VeeamClient { id = split[1].trim(); } else if (part.matches("CreationTime(\\s)+:(.)*")) { String [] split = part.split(":", 2); - created = split[1].trim(); + split[1] = StringUtils.trim(split[1]); + String [] time = split[1].split("[:/ ]"); + Calendar cal = Calendar.getInstance(); + cal.set(Integer.parseInt(time[2]), Integer.parseInt(time[0]) - 1, Integer.parseInt(time[1]), Integer.parseInt(time[3]), Integer.parseInt(time[4]), Integer.parseInt(time[5])); + created = cal.getTime(); } else if (part.matches("Type(\\s)+:(.)*")) { String [] split = part.split(":"); type = split[1].trim(); diff --git a/ui/src/components/view/ListResourceTable.vue b/ui/src/components/view/ListResourceTable.vue index c97aa0fb17e..16c1b388c5d 100644 --- a/ui/src/components/view/ListResourceTable.vue +++ b/ui/src/components/view/ListResourceTable.vue @@ -51,6 +51,11 @@ {{ text }} + + +
From f7345e861f832dbf5576c24c21eef73d72fe5d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jandre?= <48719461+JoaoJandre@users.noreply.github.com> Date: Fri, 18 Aug 2023 06:12:06 -0300 Subject: [PATCH 14/31] cpvm: Fix CPVM not releasing CPU after closing console (#7826) --- .../cloud/consoleproxy/ConsoleProxyNoVncClient.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java index 2dfea2251fb..27fabb53124 100644 --- a/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java +++ b/services/console-proxy/server/src/main/java/com/cloud/consoleproxy/ConsoleProxyNoVncClient.java @@ -113,30 +113,34 @@ public class ConsoleProxyNoVncClient implements ConsoleProxyClient { if (client.isVncOverWebSocketConnectionOpen()) { updateFrontEndActivityTime(); } - connectionAlive = client.isVncOverWebSocketConnectionAlive(); + connectionAlive = session.isOpen(); try { Thread.sleep(1); - } catch (Exception e) { - s_logger.warn("Error on sleep for vnc over websocket", e); + } catch (InterruptedException e) { + s_logger.error("Error on sleep for vnc over websocket", e); } } else if (client.isVncOverNioSocket()) { byte[] bytesArr; int nextBytes = client.getNextBytes(); bytesArr = new byte[nextBytes]; client.readBytes(bytesArr, nextBytes); + s_logger.trace(String.format("Read [%s] bytes from client [%s]", nextBytes, clientId)); if (nextBytes > 0) { session.getRemote().sendBytes(ByteBuffer.wrap(bytesArr)); updateFrontEndActivityTime(); + } else { + connectionAlive = session.isOpen(); } } else { b = new byte[100]; readBytes = client.read(b); + s_logger.trace(String.format("Read [%s] bytes from client [%s]", readBytes, clientId)); if (readBytes == -1 || (readBytes > 0 && !sendReadBytesToNoVNC(b, readBytes))) { connectionAlive = false; } } } - connectionAlive = false; + s_logger.info(String.format("Connection with client [%s] is dead.", clientId)); } catch (IOException e) { s_logger.error("Error on VNC client", e); } From 9083a677ae132c5ce4da2f73b4f04ec2356b33b5 Mon Sep 17 00:00:00 2001 From: sato03 Date: Sun, 20 Aug 2023 10:41:07 -0300 Subject: [PATCH 15/31] systeminstances field (#7871) Co-authored-by: Henrique Sato --- .../cloudstack/metrics/MetricsServiceImpl.java | 12 ++++++++++-- .../cloudstack/response/HostMetricsResponse.java | 14 ++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index 979e4aacbe7..a27896aa58d 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -682,8 +682,10 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements final Float cpuDisableThreshold = DeploymentClusterPlanner.ClusterCPUCapacityDisableThreshold.valueIn(clusterId); final Float memoryDisableThreshold = DeploymentClusterPlanner.ClusterMemoryCapacityDisableThreshold.valueIn(clusterId); - Long upInstances = 0L; - Long totalInstances = 0L; + long upInstances = 0L; + long totalInstances = 0L; + long upSystemInstances = 0L; + long totalSystemInstances = 0L; for (final VMInstanceVO instance: vmInstanceDao.listByHostId(hostId)) { if (instance == null) { continue; @@ -693,10 +695,16 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements if (instance.getState() == VirtualMachine.State.Running) { upInstances++; } + } else if (instance.getType().isUsedBySystem()) { + totalSystemInstances++; + if (instance.getState() == VirtualMachine.State.Running) { + upSystemInstances++; + } } } metricsResponse.setPowerState(hostResponse.getOutOfBandManagementResponse().getPowerState()); metricsResponse.setInstances(upInstances, totalInstances); + metricsResponse.setSystemInstances(upSystemInstances, totalSystemInstances); metricsResponse.setCpuTotal(hostResponse.getCpuNumber(), hostResponse.getCpuSpeed()); metricsResponse.setCpuUsed(hostResponse.getCpuUsed(), hostResponse.getCpuNumber(), hostResponse.getCpuSpeed()); metricsResponse.setCpuAllocated(hostResponse.getCpuAllocated(), hostResponse.getCpuNumber(), hostResponse.getCpuSpeed()); diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsResponse.java b/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsResponse.java index 30f7e06ce58..8e12d2cbb6d 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsResponse.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/response/HostMetricsResponse.java @@ -36,6 +36,10 @@ public class HostMetricsResponse extends HostResponse { @Param(description = "instances on the host") private String instances; + @SerializedName("systeminstances") + @Param(description = "system vm instances on the host") + private String systemInstances; + @SerializedName("cputotalghz") @Param(description = "the total cpu capacity in Ghz") private String cpuTotal; @@ -108,10 +112,12 @@ public class HostMetricsResponse extends HostResponse { this.powerState = powerState; } - public void setInstances(final Long running, final Long total) { - if (running != null && total != null) { - this.instances = String.format("%d / %d", running, total); - } + public void setSystemInstances(final long running, final long total) { + this.systemInstances = String.format("%d / %d", running, total); + } + + public void setInstances(final long running, final long total) { + this.instances = String.format("%d / %d", running, total); } public void setCpuTotal(final Integer cpuNumber, final Long cpuSpeed) { From d296f54c7f228ced8b6f25a19cb6ade6e0fc9212 Mon Sep 17 00:00:00 2001 From: Sina Kashipazha Date: Sun, 20 Aug 2023 15:44:31 +0200 Subject: [PATCH 16/31] Api: update command domainId/account descriptions (#7876) --- .../cloudstack/api/command/user/network/CreateNetworkCmd.java | 4 +++- .../api/command/user/snapshot/CreateSnapshotCmd.java | 2 +- .../apache/cloudstack/api/command/user/vm/DeployVMCmd.java | 2 +- .../cloudstack/api/command/user/volume/CreateVolumeCmd.java | 3 ++- .../cloudstack/api/command/user/volume/UploadVolumeCmd.java | 2 +- .../org/apache/cloudstack/api/response/UserVmResponse.java | 2 +- .../api/command/admin/ratelimit/ResetApiLimitCmd.java | 2 +- .../tungsten/api/command/ListTungstenFabricTagCmd.java | 2 +- .../storage/datastore/util/ListElastistorVolumeResponse.java | 2 +- .../cloudstack/api/response/LdapConfigurationResponse.java | 2 +- .../cloudstack/api/response/LinkAccountToLdapResponse.java | 2 +- .../cloudstack/api/response/LinkDomainToLdapResponse.java | 2 +- .../com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java | 2 +- 13 files changed, 16 insertions(+), 13 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index 5b814e77004..3bc90e5e8ae 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -118,7 +118,9 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { private Long projectId; @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "domain ID of the account owning a network. " + - "If no account is provided then network will be assigned to the caller account and domain") + "If the account is not specified, but the acltype is Account or not specified, the network will be automatically assigned to the caller account and domain. " + + "To create a network under the domain without linking it to any account, make sure to include acltype=Domain parameter in the api call. " + + "If account is not specified, but acltype is Domain, the network will be created for the specified domain.") private Long domainId; @Parameter(name = ApiConstants.SUBDOMAIN_ACCESS, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java index aee72b3dbae..56e112963a9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/CreateSnapshotCmd.java @@ -62,7 +62,7 @@ public class CreateSnapshotCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, - description = "The domain ID of the snapshot. If used with the account parameter, specifies a domain for the account associated with the disk volume.") + description = "The domain ID of the snapshot. If used with the account parameter, specifies a domain for the account associated with the disk volume. If account is NOT provided then snapshot will be assigned to the caller account and domain.") private Long domainId; @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.UUID, entityType = VolumeResponse.class, required = true, description = "The ID of the disk volume") diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 7aa69bd11ee..12212dac417 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -109,7 +109,7 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the virtual machine. Must be used with domainId.") private String accountName; - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. If account is NOT provided then virtual machine will be assigned to the caller account and domain.") private Long domainId; //Network information diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 3426a95686d..566e8a46bd9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -70,7 +70,8 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC type = CommandType.UUID, entityType = DomainResponse.class, description = "the domain ID associated with the disk offering. If used with the account parameter" - + " returns the disk volume associated with the account for the specified domain.") + + " returns the disk volume associated with the account for the specified domain." + + "If account is NOT provided then the volume will be assigned to the caller account and domain.") private Long domainId; @Parameter(name = ApiConstants.DISK_OFFERING_ID, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java index b40761d123a..c622081079d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/UploadVolumeCmd.java @@ -78,7 +78,7 @@ public class UploadVolumeCmd extends BaseAsyncCmd implements UserCmd { @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, - description = "an optional domainId. If the account parameter is used, domainId must also be used.") + description = "an optional domainId. If the account parameter is used, domainId must also be used. If account is NOT provided then volume will be assigned to the caller account and domain.") private Long domainId; @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional accountName. Must be used with domainId.") diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index f2903b2626c..114403da7bc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -103,7 +103,7 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co private String group; @SerializedName(ApiConstants.ZONE_ID) - @Param(description = "the ID of the availablility zone for the virtual machine") + @Param(description = "the ID of the availability zone for the virtual machine") private String zoneId; @SerializedName(ApiConstants.ZONE_NAME) diff --git a/plugins/api/rate-limit/src/main/java/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java b/plugins/api/rate-limit/src/main/java/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java index 7aca752252f..5be0dfcb678 100644 --- a/plugins/api/rate-limit/src/main/java/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java +++ b/plugins/api/rate-limit/src/main/java/org/apache/cloudstack/api/command/admin/ratelimit/ResetApiLimitCmd.java @@ -53,7 +53,7 @@ public class ResetApiLimitCmd extends BaseCmd { ///////////////////////////////////////////////////// @ACL - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.UUID, entityType = AccountResponse.class, description = "the ID of the acount whose limit to be reset") + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.UUID, entityType = AccountResponse.class, description = "the ID of the account whose limit to be reset") private Long accountId; ///////////////////////////////////////////////////// diff --git a/plugins/network-elements/tungsten/src/main/java/org/apache/cloudstack/network/tungsten/api/command/ListTungstenFabricTagCmd.java b/plugins/network-elements/tungsten/src/main/java/org/apache/cloudstack/network/tungsten/api/command/ListTungstenFabricTagCmd.java index dd64587ac14..657bc431bc4 100644 --- a/plugins/network-elements/tungsten/src/main/java/org/apache/cloudstack/network/tungsten/api/command/ListTungstenFabricTagCmd.java +++ b/plugins/network-elements/tungsten/src/main/java/org/apache/cloudstack/network/tungsten/api/command/ListTungstenFabricTagCmd.java @@ -61,7 +61,7 @@ public class ListTungstenFabricTagCmd extends BaseListCmd { @Parameter(name = ApiConstants.VM_UUID, type = CommandType.STRING, description = "the uuid of Tungsten-Fabric vm") private String vmUuid; - @Parameter(name = ApiConstants.NIC_UUID, type = CommandType.STRING, description = "tthe uuid of Tungsten-Fabric nic") + @Parameter(name = ApiConstants.NIC_UUID, type = CommandType.STRING, description = "the uuid of Tungsten-Fabric nic") private String nicUuid; @Parameter(name = ApiConstants.POLICY_UUID, type = CommandType.STRING, description = "the uuid of Tungsten-Fabric policy") diff --git a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ListElastistorVolumeResponse.java b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ListElastistorVolumeResponse.java index b251d7a3a3a..e141cc1dffc 100644 --- a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ListElastistorVolumeResponse.java +++ b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/util/ListElastistorVolumeResponse.java @@ -48,7 +48,7 @@ public class ListElastistorVolumeResponse extends BaseResponse { private String compression; @SerializedName("sync") - @Param(description = "syncronization") + @Param(description = "synchronization") private String sync; public String getGraceAllowed() { diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LdapConfigurationResponse.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LdapConfigurationResponse.java index 5c6fc8a0258..744c73d8e77 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LdapConfigurationResponse.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LdapConfigurationResponse.java @@ -32,7 +32,7 @@ public class LdapConfigurationResponse extends BaseResponse { private String hostname; @SerializedName(ApiConstants.PORT) - @Param(description = "port teh ldap server is running on") + @Param(description = "port the ldap server is running on") private int port; @SerializedName(ApiConstants.DOMAIN_ID) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkAccountToLdapResponse.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkAccountToLdapResponse.java index 8dacfb7ca3e..6273273bdbf 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkAccountToLdapResponse.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkAccountToLdapResponse.java @@ -34,7 +34,7 @@ public class LinkAccountToLdapResponse extends BaseResponse { private String ldapDomain; @SerializedName(ApiConstants.TYPE) - @Param(description = "type of the name in LDAP which is linke to the domain") + @Param(description = "type of the name in LDAP which is linked to the domain") private String type; @SerializedName(ApiConstants.ACCOUNT_TYPE) diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkDomainToLdapResponse.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkDomainToLdapResponse.java index bc552ee2cf6..be057fd8418 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkDomainToLdapResponse.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/api/response/LinkDomainToLdapResponse.java @@ -39,7 +39,7 @@ public class LinkDomainToLdapResponse extends BaseResponse { private String ldapDomain; @SerializedName(ApiConstants.TYPE) - @Param(description = "type of the name in LDAP which is linke to the domain") + @Param(description = "type of the name in LDAP which is linked to the domain") private String type; @SerializedName(ApiConstants.ACCOUNT_TYPE) diff --git a/server/src/main/java/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java b/server/src/main/java/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java index 53f45c5956c..63385e22e32 100644 --- a/server/src/main/java/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java +++ b/server/src/main/java/com/cloud/api/auth/DefaultLoginAPIAuthenticatorCmd.java @@ -62,7 +62,7 @@ public class DefaultLoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthe @Parameter(name = ApiConstants.DOMAIN, type = CommandType.STRING, description = "Path of the domain that the user belongs to. Example: domain=/com/cloud/internal. If no domain is passed in, the ROOT (/) domain is assumed.") private String domain; - @Parameter(name = ApiConstants.DOMAIN__ID, type = CommandType.LONG, description = "The id of the domain that the user belongs to. If both domain and domainId are passed in, \"domainId\" parameter takes precendence") + @Parameter(name = ApiConstants.DOMAIN__ID, type = CommandType.LONG, description = "The id of the domain that the user belongs to. If both domain and domainId are passed in, \"domainId\" parameter takes precedence.") private Long domainId; @Inject From 0b4afedb5b52a5ba7747eb17169cc856eb9a9459 Mon Sep 17 00:00:00 2001 From: John Bampton Date: Sun, 20 Aug 2023 23:49:07 +1000 Subject: [PATCH 17/31] pre-commit: add hook to check for the existence of private keys (#7805) --- .pre-commit-config.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 73298a1400d..f7c3c0f90e1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,20 @@ repos: #- id: check-merge-conflict - id: check-vcs-permalinks #- id: check-yaml - #- id: detect-private-key + - id: detect-private-key + exclude: > + (?x) + ^scripts/vm/systemvm/id_rsa\.cloud$| + ^server/src/test/java/com/cloud/keystore/KeystoreTest\.java$| + ^server/src/test/resources/certs/dsa_self_signed\.key$| + ^server/src/test/resources/certs/non_root\.key$| + ^server/src/test/resources/certs/root_chain\.key$| + ^server/src/test/resources/certs/rsa_ca_signed\.key$| + ^server/src/test/resources/certs/rsa_self_signed_with_pwd\.key$| + ^server/src/test/resources/certs/rsa_self_signed\.key$| + ^services/console-proxy/rdpconsole/src/test/doc/rdp-key\.pem$| + ^systemvm/agent/certs/localhost\.key$| + ^systemvm/agent/certs/realhostip\.key$ - id: end-of-file-fixer exclude: \.vhd$ #- id: fix-byte-order-marker From ddc2a362a88eb5f4347693a9c1ec9ba8922a4b77 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Mon, 21 Aug 2023 06:13:25 -0300 Subject: [PATCH 18/31] UI: Add missing tooltips to Register/Upload Templates/ISOs views (#7879) --- ui/src/views/image/RegisterOrUploadIso.vue | 60 ++++++++++++++----- .../views/image/RegisterOrUploadTemplate.vue | 60 ++++++++++++++----- 2 files changed, 91 insertions(+), 29 deletions(-) diff --git a/ui/src/views/image/RegisterOrUploadIso.vue b/ui/src/views/image/RegisterOrUploadIso.vue index cf1435a6034..aa127a3d405 100644 --- a/ui/src/views/image/RegisterOrUploadIso.vue +++ b/ui/src/views/image/RegisterOrUploadIso.vue @@ -35,8 +35,10 @@ + name="url"> + - + + - + + - + + - + + - + + - + + + ref="userdataid"> + - + + + - + + @@ -276,7 +306,7 @@ export default { params.showicon = true this.zoneLoading = true - if (store.getters.userInfo.roletype === this.rootAdmin) { + if (store.getters.userInfo.roletype === 'Admin') { this.allowed = true } api('listZones', params).then(json => { diff --git a/ui/src/views/image/RegisterOrUploadTemplate.vue b/ui/src/views/image/RegisterOrUploadTemplate.vue index e4d4687c326..c8e7ad3510e 100644 --- a/ui/src/views/image/RegisterOrUploadTemplate.vue +++ b/ui/src/views/image/RegisterOrUploadTemplate.vue @@ -32,7 +32,10 @@ @finish="handleSubmit" layout="vertical">
- + +
- + + - + +
+
+ - + + - + + - + + - + + @@ -174,7 +199,10 @@ - + + + + ref="userdataid"> + Date: Mon, 21 Aug 2023 10:48:33 -0300 Subject: [PATCH 19/31] server: Allow admins to disable the 2FA of users in subdomains (#7870) --- .../java/com/cloud/user/AccountManagerImpl.java | 7 ++----- .../java/com/cloud/user/AccountManagerImplTest.java | 13 ++++++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index c20e2fc2abf..99896dc9827 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -3327,7 +3327,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M protected UserTwoFactorAuthenticationSetupResponse disableTwoFactorAuthentication(Long userId, Account caller, Account owner) { UserVO userVO = null; if (userId != null) { - userVO = validateUser(userId, caller.getDomainId()); + userVO = validateUser(userId); owner = _accountService.getActiveAccountById(userVO.getAccountId()); } else { userId = CallContext.current().getCallingUserId(); @@ -3349,16 +3349,13 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M return response; } - private UserVO validateUser(Long userId, Long domainId) { + private UserVO validateUser(Long userId) { UserVO user = null; if (userId != null) { user = _userDao.findById(userId); if (user == null) { throw new InvalidParameterValueException("Invalid user ID provided"); } - if (_accountDao.findById(user.getAccountId()).getDomainId() != domainId) { - throw new InvalidParameterValueException("User doesn't belong to the specified account or domain"); - } } return user; } diff --git a/server/src/test/java/com/cloud/user/AccountManagerImplTest.java b/server/src/test/java/com/cloud/user/AccountManagerImplTest.java index c79b5069c2d..2f3a68e20af 100644 --- a/server/src/test/java/com/cloud/user/AccountManagerImplTest.java +++ b/server/src/test/java/com/cloud/user/AccountManagerImplTest.java @@ -875,19 +875,17 @@ public class AccountManagerImplTest extends AccountManagetImplTestBase { @Test public void testDisableUserTwoFactorAuthentication() { Long userId = 1L; + Long accountId = 2L; UserVO userVO = Mockito.mock(UserVO.class); Account caller = Mockito.mock(Account.class); + Account owner = Mockito.mock(Account.class); - AccountVO accountMock = Mockito.mock(AccountVO.class); Mockito.doNothing().when(accountManagerImpl).checkAccess(nullable(Account.class), Mockito.isNull(), nullable(Boolean.class), nullable(Account.class)); - Mockito.when(caller.getDomainId()).thenReturn(1L); Mockito.when(userDaoMock.findById(userId)).thenReturn(userVO); - Mockito.when(userVO.getAccountId()).thenReturn(1L); - Mockito.when(_accountDao.findById(1L)).thenReturn(accountMock); - Mockito.when(accountMock.getDomainId()).thenReturn(1L); - Mockito.when(_accountService.getActiveAccountById(1L)).thenReturn(caller); + Mockito.when(userVO.getAccountId()).thenReturn(accountId); + Mockito.when(_accountService.getActiveAccountById(accountId)).thenReturn(owner); userVoMock.setKeyFor2fa("EUJEAEDVOURFZTE6OGWVTJZMI54QGMIL"); userVoMock.setUser2faProvider("totp"); @@ -895,8 +893,9 @@ public class AccountManagerImplTest extends AccountManagetImplTestBase { Mockito.when(userDaoMock.createForUpdate()).thenReturn(userVoMock); - UserTwoFactorAuthenticationSetupResponse response = accountManagerImpl.disableTwoFactorAuthentication(userId, caller, caller); + UserTwoFactorAuthenticationSetupResponse response = accountManagerImpl.disableTwoFactorAuthentication(userId, caller, owner); + Mockito.verify(accountManagerImpl).checkAccess(caller, null, true, owner); Assert.assertNull(response.getSecretCode()); Assert.assertNull(userVoMock.getKeyFor2fa()); Assert.assertNull(userVoMock.getUser2faProvider()); From 6cded7e010207a4cee6df75319e6a90638e93afd Mon Sep 17 00:00:00 2001 From: Marcus Sorensen Date: Tue, 22 Aug 2023 03:01:20 -0600 Subject: [PATCH 20/31] Support GNU sed apidoc for MacOS build (#7888) Co-authored-by: Marcus Sorensen --- tools/apidoc/build-apidoc.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/apidoc/build-apidoc.sh b/tools/apidoc/build-apidoc.sh index 21cceb0a282..35551cda597 100755 --- a/tools/apidoc/build-apidoc.sh +++ b/tools/apidoc/build-apidoc.sh @@ -59,8 +59,12 @@ fi # Default case for Linux sed, just use "-i" sedi='-i' case "$(uname)" in - # For macOS, use two parameters - Darwin*) sedi='-i ""' + # For macOS, use two parameters, if gnu sed is not set up + Darwin*) + if ! $(which sed | grep -q gnu); then + sedi='-i ""' + fi + ;; esac set -e From 0bd7462c54acae87f723b36367f03fb8cfa4162c Mon Sep 17 00:00:00 2001 From: John Bampton Date: Tue, 22 Aug 2023 19:01:39 +1000 Subject: [PATCH 21/31] Link README logo to website (#7884) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 57cd8b83379..2519c6cd273 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Apache CloudStack [![Build Status](https://github.com/apache/cloudstack/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/build.yml) [![UI Build](https://github.com/apache/cloudstack/actions/workflows/ui.yml/badge.svg)](https://github.com/apache/cloudstack/actions/workflows/ui.yml) [![License Check](https://github.com/apache/cloudstack/actions/workflows/rat.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/rat.yml) [![Simulator CI](https://github.com/apache/cloudstack/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/ci.yml) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache_cloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache_cloudstack) [![codecov](https://codecov.io/gh/apache/cloudstack/branch/main/graph/badge.svg)](https://codecov.io/gh/apache/cloudstack) -![Apache CloudStack](tools/logo/apache_cloudstack.png) +[![Apache CloudStack](tools/logo/apache_cloudstack.png)](https://cloudstack.apache.org/) Apache CloudStack is open source software designed to deploy and manage large networks of virtual machines, as a highly available, highly scalable From 405ef82aefd5991ea8d0bf89081d7647b7ed9a89 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Tue, 22 Aug 2023 06:02:54 -0300 Subject: [PATCH 22/31] UI: Fix hide delete button for templates that are in Installing template state (#7882) This PR hides the Delete Template button when a template is installing Fixes: #7865 --- ui/src/views/image/TemplateZones.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/image/TemplateZones.vue b/ui/src/views/image/TemplateZones.vue index a2f8834be6e..edef387e2be 100644 --- a/ui/src/views/image/TemplateZones.vue +++ b/ui/src/views/image/TemplateZones.vue @@ -66,7 +66,7 @@ @onClick="showCopyTemplate(record)" /> Date: Tue, 22 Aug 2023 11:07:16 +0200 Subject: [PATCH 23/31] AutoScaling: support Managed User Data (#7769) --- .../cloud/network/as/AutoScaleVmProfile.java | 4 + .../org/apache/cloudstack/api/BaseCmd.java | 17 ++ .../CreateAutoScaleVmProfileCmd.java | 15 + .../UpdateAutoScaleVmProfileCmd.java | 17 ++ .../api/command/user/vm/DeployVMCmd.java | 26 +- .../command/user/vm/ResetVMUserDataCmd.java | 16 +- .../api/command/user/vm/ScaleVMCmd.java | 15 +- .../api/command/user/vm/UpdateVMCmd.java | 14 +- .../api/command/user/vm/UpgradeVMCmd.java | 15 +- .../response/AutoScaleVmProfileResponse.java | 48 ++++ .../network/as/AutoScaleVmProfileVO.java | 24 ++ .../upgrade/dao/Upgrade41800to41810.java | 5 + .../META-INF/db/schema-41800to41810.sql | 4 + .../network/as/AutoScaleVmProfileVOTest.java | 31 +++ .../com/cloud/api/ApiAsyncJobDispatcher.java | 6 + .../java/com/cloud/api/ApiResponseHelper.java | 14 + .../main/java/com/cloud/api/ApiServer.java | 3 + .../network/as/AutoScaleManagerImpl.java | 53 +++- .../main/java/com/cloud/vm/UserVmManager.java | 4 + .../java/com/cloud/vm/UserVmManagerImpl.java | 6 +- .../com/cloud/api/ApiResponseHelperTest.java | 72 +++++ .../network/as/AutoScaleManagerImplTest.java | 75 ++++- test/integration/smoke/test_vm_autoscaling.py | 69 +++++ ui/public/locales/en.json | 1 + ui/src/views/compute/AutoScaleVmProfile.vue | 92 ++++++- .../views/compute/CreateAutoScaleVmGroup.vue | 257 +++++++++++++++++- ui/src/views/compute/ResetUserData.vue | 10 +- 27 files changed, 785 insertions(+), 128 deletions(-) diff --git a/api/src/main/java/com/cloud/network/as/AutoScaleVmProfile.java b/api/src/main/java/com/cloud/network/as/AutoScaleVmProfile.java index afeba0bd749..68d18a0edc9 100644 --- a/api/src/main/java/com/cloud/network/as/AutoScaleVmProfile.java +++ b/api/src/main/java/com/cloud/network/as/AutoScaleVmProfile.java @@ -35,6 +35,10 @@ public interface AutoScaleVmProfile extends ControlledEntity, InternalIdentity, String getUserData(); + Long getUserDataId(); + + String getUserDataDetails(); + public String getUuid(); public Long getZoneId(); diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index e431f63557c..79e103a291d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -21,8 +21,10 @@ import java.lang.reflect.Field; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -42,6 +44,7 @@ import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.ImageStoreService; import org.apache.cloudstack.usage.UsageService; +import org.apache.commons.collections.MapUtils; import org.apache.log4j.Logger; import com.cloud.configuration.ConfigurationService; @@ -456,4 +459,18 @@ public abstract class BaseCmd { return ApiCommandResourceType.None; } + public Map convertDetailsToMap(Map details) { + Map detailsMap = new HashMap(); + if (MapUtils.isNotEmpty(details)) { + Collection parameterCollection = details.values(); + Iterator iter = parameterCollection.iterator(); + while (iter.hasNext()) { + HashMap value = (HashMap)iter.next(); + for (Map.Entry entry: value.entrySet()) { + detailsMap.put(entry.getKey(),entry.getValue()); + } + } + } + return detailsMap; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java index ba7149ea3d8..db6ccd9ce53 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; @@ -107,6 +108,12 @@ public class CreateAutoScaleVmProfileCmd extends BaseAsyncCreateCmd { since = "4.18.0") private String userData; + @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.18.1") + private Long userDataId; + + @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", since = "4.18.1") + private Map userDataDetails; + @Parameter(name = ApiConstants.AUTOSCALE_USER_ID, type = CommandType.UUID, entityType = UserResponse.class, @@ -163,6 +170,14 @@ public class CreateAutoScaleVmProfileCmd extends BaseAsyncCreateCmd { return userData; } + public Long getUserDataId() { + return userDataId; + } + + public Map getUserDataDetails() { + return convertDetailsToMap(userDataDetails); + } + public Long getAutoscaleUserId() { return autoscaleUserId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java index c0b385932a9..3e65d38e520 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java @@ -35,6 +35,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.context.CallContext; @@ -102,6 +103,14 @@ public class UpdateAutoScaleVmProfileCmd extends BaseAsyncCustomIdCmd { since = "4.18.0") private String userData; + @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the userdata", + since = "4.18.1") + private Long userDataId; + + @Parameter(name = ApiConstants.USER_DATA_DETAILS, type = CommandType.MAP, description = "used to specify the parameters values for the variables in userdata.", + since = "4.18.1") + private Map userDataDetails; + @Parameter(name = ApiConstants.AUTOSCALE_USER_ID, type = CommandType.UUID, entityType = UserResponse.class, @@ -156,6 +165,14 @@ public class UpdateAutoScaleVmProfileCmd extends BaseAsyncCustomIdCmd { return userData; } + public Long getUserDataId() { + return userDataId; + } + + public Map getUserDataDetails() { + return convertDetailsToMap(userDataDetails); + } + public Long getAutoscaleUserId() { return autoscaleUserId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 12212dac417..ef6dba63166 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -309,17 +309,8 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG } public Map getDetails() { - Map customparameterMap = new HashMap(); - if (details != null && details.size() != 0) { - Collection parameterCollection = details.values(); - Iterator iter = parameterCollection.iterator(); - while (iter.hasNext()) { - HashMap value = (HashMap)iter.next(); - for (Map.Entry entry: value.entrySet()) { - customparameterMap.put(entry.getKey(),entry.getValue()); - } - } - } + Map customparameterMap = convertDetailsToMap(details); + if (getBootType() != null) { customparameterMap.put(getBootType().toString(), getBootMode().toString()); } @@ -450,18 +441,7 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG } public Map getUserdataDetails() { - Map userdataDetailsMap = new HashMap(); - if (userdataDetails != null && userdataDetails.size() != 0) { - Collection parameterCollection = userdataDetails.values(); - Iterator iter = parameterCollection.iterator(); - while (iter.hasNext()) { - HashMap value = (HashMap)iter.next(); - for (Map.Entry entry: value.entrySet()) { - userdataDetailsMap.put(entry.getKey(),entry.getValue()); - } - } - } - return userdataDetailsMap; + return convertDetailsToMap(userdataDetails); } public Long getZoneId() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java index ad0592ca4ac..3ead67e2106 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java @@ -39,9 +39,6 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; import java.util.Map; @APICommand(name = "resetUserDataForVirtualMachine", responseObject = UserVmResponse.class, description = "Resets the UserData for virtual machine. " + @@ -117,18 +114,7 @@ public class ResetVMUserDataCmd extends BaseCmd implements UserCmd { } public Map getUserdataDetails() { - Map userdataDetailsMap = new HashMap(); - if (userdataDetails != null && userdataDetails.size() != 0) { - Collection parameterCollection = userdataDetails.values(); - Iterator iter = parameterCollection.iterator(); - while (iter.hasNext()) { - HashMap value = (HashMap)iter.next(); - for (Map.Entry entry: value.entrySet()) { - userdataDetailsMap.put(entry.getKey(),entry.getValue()); - } - } - } - return userdataDetailsMap; + return convertDetailsToMap(userdataDetails); } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ScaleVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ScaleVMCmd.java index e60bce2810a..5af45762ece 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ScaleVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ScaleVMCmd.java @@ -16,9 +16,6 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -99,17 +96,7 @@ public class ScaleVMCmd extends BaseAsyncCmd implements UserCmd { //it is because details.values() cannot be cast to a map. //it gives a exception public Map getDetails() { - Map customparameterMap = new HashMap(); - if (details != null && details.size() != 0) { - Collection parameterCollection = details.values(); - Iterator iter = parameterCollection.iterator(); - while (iter.hasNext()) { - HashMap value = (HashMap)iter.next(); - for (String key : value.keySet()) { - customparameterMap.put(key, value.get(key)); - } - } - } + Map customparameterMap = convertDetailsToMap(details); if (shrinkOk != null) customparameterMap.put(ApiConstants.SHRINK_OK, String.valueOf(isShrinkOk())); if (autoMigrate != null) customparameterMap.put(ApiConstants.AUTO_MIGRATE, String.valueOf(getAutoMigrate())); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index 3e0ef75ecdd..32ce1f6db52 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -18,7 +18,6 @@ package org.apache.cloudstack.api.command.user.vm; import java.util.Collection; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -176,18 +175,7 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction, } public Map getUserdataDetails() { - Map userdataDetailsMap = new HashMap(); - if (userdataDetails != null && userdataDetails.size() != 0) { - Collection parameterCollection = userdataDetails.values(); - Iterator iter = parameterCollection.iterator(); - while (iter.hasNext()) { - HashMap value = (HashMap)iter.next(); - for (Map.Entry entry: value.entrySet()) { - userdataDetailsMap.put(entry.getKey(),entry.getValue()); - } - } - } - return userdataDetailsMap; + return convertDetailsToMap(userdataDetails); } public Boolean getDisplayVm() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpgradeVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpgradeVMCmd.java index b6acd71b4c4..4b31c12ec0a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpgradeVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpgradeVMCmd.java @@ -16,9 +16,6 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import org.apache.log4j.Logger; @@ -95,17 +92,7 @@ public class UpgradeVMCmd extends BaseCmd implements UserCmd { } public Map getDetails() { - Map customparameterMap = new HashMap(); - if (details != null && details.size() != 0) { - Collection parameterCollection = details.values(); - Iterator iter = parameterCollection.iterator(); - while (iter.hasNext()) { - HashMap value = (HashMap)iter.next(); - for (String key : value.keySet()) { - customparameterMap.put(key, value.get(key)); - } - } - } + Map customparameterMap = convertDetailsToMap(details); if (shrinkOk != null) customparameterMap.put(ApiConstants.SHRINK_OK, String.valueOf(isShrinkOk())); if (autoMigrate != null) customparameterMap.put(ApiConstants.AUTO_MIGRATE, String.valueOf(getAutoMigrate())); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java index 66d7296f406..9f238344730 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java @@ -72,6 +72,18 @@ public class AutoScaleVmProfileResponse extends BaseResponse implements Controll @Param(description = "Base 64 encoded VM user data") private String userData; + @SerializedName(ApiConstants.USER_DATA_ID) @Param(description="the id of userdata used for the VM", since = "4.18.1") + private String userDataId; + + @SerializedName(ApiConstants.USER_DATA_NAME) @Param(description="the name of userdata used for the VM", since = "4.18.1") + private String userDataName; + + @SerializedName(ApiConstants.USER_DATA_POLICY) @Param(description="the userdata override policy with the userdata provided while deploying VM", since = "4.18.1") + private String userDataPolicy; + + @SerializedName(ApiConstants.USER_DATA_DETAILS) @Param(description="list of variables and values for the variables declared in userdata", since = "4.18.1") + private String userDataDetails; + @SerializedName(ApiConstants.AUTOSCALE_USER_ID) @Param(description = "the ID of the user used to launch and destroy the VMs") private String autoscaleUserId; @@ -153,6 +165,22 @@ public class AutoScaleVmProfileResponse extends BaseResponse implements Controll this.userData = userData; } + public void setUserDataId(String userDataId) { + this.userDataId = userDataId; + } + + public void setUserDataName(String userDataName) { + this.userDataName = userDataName; + } + + public void setUserDataPolicy(String userDataPolicy) { + this.userDataPolicy = userDataPolicy; + } + + public void setUserDataDetails(String userDataDetails) { + this.userDataDetails = userDataDetails; + } + @Override public void setAccountName(String accountName) { this.accountName = accountName; @@ -193,4 +221,24 @@ public class AutoScaleVmProfileResponse extends BaseResponse implements Controll public void setForDisplay(Boolean forDisplay) { this.forDisplay = forDisplay; } + + public String getUserData() { + return userData; + } + + public String getUserDataId() { + return userDataId; + } + + public String getUserDataName() { + return userDataName; + } + + public String getUserDataPolicy() { + return userDataPolicy; + } + + public String getUserDataDetails() { + return userDataDetails; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmProfileVO.java b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmProfileVO.java index 0a0ba1e62aa..21291062756 100644 --- a/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmProfileVO.java +++ b/engine/schema/src/main/java/com/cloud/network/as/AutoScaleVmProfileVO.java @@ -88,6 +88,12 @@ public class AutoScaleVmProfileVO implements AutoScaleVmProfile, Identity, Inter @Basic(fetch = FetchType.LAZY) private String userData; + @Column(name = "user_data_id", nullable = true) + private Long userDataId = null; + + @Column(name = "user_data_details", updatable = true, length = 4096) + private String userDataDetails; + @Column(name = GenericDao.REMOVED_COLUMN) protected Date removed; @@ -228,6 +234,24 @@ public class AutoScaleVmProfileVO implements AutoScaleVmProfile, Identity, Inter return userData; } + @Override + public Long getUserDataId() { + return userDataId; + } + + public void setUserDataId(Long userDataId) { + this.userDataId = userDataId; + } + + @Override + public String getUserDataDetails() { + return userDataDetails; + } + + public void setUserDataDetails(String userDataDetails) { + this.userDataDetails = userDataDetails; + } + @Override public String getUuid() { return uuid; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index a2733215060..53bd497ca22 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -63,6 +63,7 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate fixForeignKeyNames(conn); updateGuestOsMappings(conn); copyGuestOsMappingsToVMware80u1(); + addForeignKeyToAutoscaleVmprofiles(conn); } @Override @@ -225,4 +226,8 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate DbUpgradeUtils.dropKeysIfExist(conn, "cloud.volumes", keys, false); DbUpgradeUtils.addForeignKey(conn, "volumes", "passphrase_id","passphrase", "id"); } + + private void addForeignKeyToAutoscaleVmprofiles(Connection conn) { + DbUpgradeUtils.addForeignKey(conn, "autoscale_vmprofiles", "user_data_id", "user_data", "id"); + } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql index e07e12871f3..495c184918c 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql @@ -35,6 +35,10 @@ CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VM CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'VMware', '8.0.0.1', 'windows2019srvNext_64Guest'); CALL ADD_GUEST_OS_AND_HYPERVISOR_MAPPING (6, 'Windows Server 2022 (64-bit)', 'Xenserver', '8.2.0', 'Windows Server 2022 (64-bit)'); +-- Support userdata ids and details in VM AutoScaling +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.autoscale_vmprofiles', 'user_data_id', 'bigint unsigned DEFAULT NULL COMMENT "id of the user data" AFTER `user_data`'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.autoscale_vmprofiles', 'user_data_details', 'mediumtext DEFAULT NULL COMMENT "value of the comma-separated list of parameters" AFTER `user_data_id`'); + -- Don't enable CPU cap for default system offerings, fixes regression from https://github.com/apache/cloudstack/pull/6420 UPDATE `cloud`.`service_offering` so SET so.limit_cpu_use = 0 diff --git a/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java b/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java index 6f49c3d4e1e..7e9658e1dd3 100755 --- a/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java +++ b/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java @@ -26,6 +26,18 @@ import org.junit.Test; public class AutoScaleVmProfileVOTest { + static long zoneId = 1L; + static long domainId = 2L; + static long accountId = 3L; + static long serviceOfferingId = 4L; + static long templateId = 5L; + static String userdata = "userdata"; + static long userdataId = 6L; + static String userdataDetails = "userdataDetails"; + static String userdataNew = "userdataNew"; + + static long autoScaleUserId = 7L; + @Test public void testCounterParamsForUpdate() { AutoScaleVmProfileVO profile = new AutoScaleVmProfileVO(); @@ -62,4 +74,23 @@ public class AutoScaleVmProfileVOTest { Assert.assertEquals("rootdisksize", otherDeployParamsList.get(1).first()); Assert.assertEquals("10", otherDeployParamsList.get(1).second()); } + + @Test + public void testProperties() { + AutoScaleVmProfileVO profile = new AutoScaleVmProfileVO(zoneId, domainId, accountId, serviceOfferingId, templateId, null, null, userdata, null, autoScaleUserId); + Assert.assertEquals(new Long(zoneId), profile.getZoneId()); + Assert.assertEquals(domainId, profile.getDomainId()); + Assert.assertEquals(accountId, profile.getAccountId()); + Assert.assertEquals(new Long(serviceOfferingId), profile.getServiceOfferingId()); + Assert.assertEquals(new Long(templateId), profile.getTemplateId()); + Assert.assertEquals(userdata, profile.getUserData()); + Assert.assertEquals(new Long(autoScaleUserId), profile.getAutoScaleUserId()); + + profile.setUserData(userdataNew); + profile.setUserDataId(userdataId); + profile.setUserDataDetails(userdataDetails); + Assert.assertEquals(userdataNew, profile.getUserData()); + Assert.assertEquals(new Long(userdataId), profile.getUserDataId()); + Assert.assertEquals(userdataDetails, profile.getUserDataDetails()); + } } diff --git a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java index b596254994c..e09e95e2ce6 100644 --- a/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java +++ b/server/src/main/java/com/cloud/api/ApiAsyncJobDispatcher.java @@ -21,6 +21,7 @@ import java.util.Map; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.BaseAsyncCreateCmd; @@ -101,6 +102,11 @@ public class ApiAsyncJobDispatcher extends AdapterBase implements AsyncJobDispat ctx.putContextParameters((Map) gson.fromJson(contextDetails, objectMapType)); } + String httpmethod = params.get(ApiConstants.HTTPMETHOD); + if (httpmethod != null) { + cmdObj.setHttpMethod(httpmethod); + } + try { // dispatch could ultimately queue the job _dispatcher.dispatch(cmdObj, params, true); diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 8fffebb3303..55f00a609e9 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -365,6 +365,7 @@ import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.user.UserData; import com.cloud.user.UserStatisticsVO; +import com.cloud.user.dao.UserDataDao; import com.cloud.user.dao.UserStatisticsDao; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; @@ -455,6 +456,8 @@ public class ApiResponseHelper implements ResponseGenerator { UserVmJoinDao userVmJoinDao; @Inject NetworkServiceMapDao ntwkSrvcDao; + @Inject + UserDataDao userDataDao; @Override public UserResponse createUserResponse(User user) { @@ -3393,9 +3396,20 @@ public class ApiResponseHelper implements ResponseGenerator { VMTemplateVO template = ApiDBUtils.findTemplateById(profile.getTemplateId()); if (template != null) { response.setTemplateId(template.getUuid()); + if (template.getUserDataOverridePolicy() != null) { + response.setUserDataPolicy(template.getUserDataOverridePolicy().toString()); + } } } response.setUserData(profile.getUserData()); + if (profile.getUserDataId() != null) { + UserData userData = userDataDao.findById(profile.getUserDataId()); + if (userData != null) { + response.setUserDataId(userData.getUuid()); + response.setUserDataName(userData.getName()); + } + } + response.setUserDataDetails(profile.getUserDataDetails()); response.setOtherDeployParams(profile.getOtherDeployParamsList()); response.setCounterParams(profile.getCounterParams()); response.setExpungeVmGracePeriod(profile.getExpungeVmGracePeriod()); diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index b62e59f5c27..e88d7cf8b53 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -740,6 +740,9 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer params.put("ctxStartEventId", String.valueOf(startEventId)); params.put("cmdEventType", asyncCmd.getEventType().toString()); params.put("ctxDetails", ApiGsonHelper.getBuilder().create().toJson(ctx.getContextParameters())); + if (asyncCmd.getHttpMethod() != null) { + params.put(ApiConstants.HTTPMETHOD, asyncCmd.getHttpMethod().toString()); + } Long instanceId = (objectId == null) ? asyncCmd.getApiResourceId() : objectId; diff --git a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java index 29cc4cead0b..de8a3ff3c83 100644 --- a/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java +++ b/server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java @@ -532,7 +532,6 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage long zoneId = cmd.getZoneId(); long serviceOfferingId = cmd.getServiceOfferingId(); Long autoscaleUserId = cmd.getAutoscaleUserId(); - String userData = cmd.getUserData(); DataCenter zone = entityMgr.findById(DataCenter.class, zoneId); @@ -545,6 +544,11 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage throw new InvalidParameterValueException("Unable to find service offering by id"); } + VirtualMachineTemplate template = entityMgr.findById(VirtualMachineTemplate.class, cmd.getTemplateId()); + if (template == null) { + throw new InvalidParameterValueException("Unable to find template by id " + cmd.getTemplateId()); + } + // validations HashMap deployParams = cmd.getDeployParamMap(); /* @@ -562,9 +566,23 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage profileVO.setDisplay(cmd.getDisplay()); } + String userData = cmd.getUserData(); + Long userDataId = cmd.getUserDataId(); + String userDataDetails = null; + if (MapUtils.isNotEmpty(cmd.getUserDataDetails())) { + userDataDetails = cmd.getUserDataDetails().toString(); + } + userData = userVmMgr.finalizeUserData(userData, userDataId, template); + userData = userVmMgr.validateUserData(userData, cmd.getHttpMethod()); if (userData != null) { profileVO.setUserData(userData); } + if (userDataId != null) { + profileVO.setUserDataId(userDataId); + } + if (userDataDetails != null) { + profileVO.setUserDataDetails(userDataDetails); + } profileVO = checkValidityAndPersist(profileVO, true); s_logger.info("Successfully create AutoScale Vm Profile with Id: " + profileVO.getId()); @@ -582,12 +600,19 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage Map> otherDeployParams = cmd.getOtherDeployParams(); Map counterParamList = cmd.getCounterParamList(); String userData = cmd.getUserData(); + Long userDataId = cmd.getUserDataId(); + String userDataDetails = null; + if (MapUtils.isNotEmpty(cmd.getUserDataDetails())) { + userDataDetails = cmd.getUserDataDetails().toString(); + } + boolean userdataUpdate = userData != null || userDataId != null || MapUtils.isNotEmpty(cmd.getUserDataDetails()); Integer expungeVmGracePeriod = cmd.getExpungeVmGracePeriod(); AutoScaleVmProfileVO vmProfile = getEntityInDatabase(CallContext.current().getCallingAccount(), "Auto Scale Vm Profile", profileId, autoScaleVmProfileDao); - boolean physicalParameterUpdate = (templateId != null || autoscaleUserId != null || counterParamList != null || otherDeployParams != null || expungeVmGracePeriod != null || userData != null); + boolean physicalParameterUpdate = (templateId != null || autoscaleUserId != null || counterParamList != null + || otherDeployParams != null || expungeVmGracePeriod != null || userdataUpdate); if (serviceOfferingId != null) { vmProfile.setServiceOfferingId(serviceOfferingId); @@ -609,10 +634,6 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage vmProfile.setCounterParamsForUpdate(counterParamList); } - if (userData != null) { - vmProfile.setUserData(userData); - } - if (expungeVmGracePeriod != null) { vmProfile.setExpungeVmGracePeriod(expungeVmGracePeriod); } @@ -625,6 +646,18 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage vmProfile.setDisplay(cmd.getDisplay()); } + if (userdataUpdate) { + if (templateId == null) { + templateId = vmProfile.getTemplateId(); + } + VirtualMachineTemplate template = entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, templateId); + userData = userVmMgr.finalizeUserData(userData, userDataId, template); + userData = userVmMgr.validateUserData(userData, cmd.getHttpMethod()); + vmProfile.setUserDataId(userDataId); + vmProfile.setUserData(userData); + vmProfile.setUserDataDetails(userDataDetails); + } + List vmGroupList = autoScaleVmGroupDao.listByAll(null, profileId); for (AutoScaleVmGroupVO vmGroupVO : vmGroupList) { if (physicalParameterUpdate && !vmGroupVO.getState().equals(AutoScaleVmGroup.State.DISABLED)) { @@ -1740,6 +1773,8 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage } String userData = profileVo.getUserData(); + Long userDataId = profileVo.getUserDataId(); + String userDataDetails = profileVo.getUserDataDetails(); UserVm vm = null; IpAddresses addrs = new IpAddresses(null, null); @@ -1763,20 +1798,20 @@ public class AutoScaleManagerImpl extends ManagerBase implements AutoScaleManage if (zone.getNetworkType() == NetworkType.Basic) { vm = userVmService.createBasicSecurityGroupVirtualMachine(zone, serviceOffering, template, null, owner, vmHostName, vmHostName, diskOfferingId, dataDiskSize, null, - hypervisorType, HTTPMethod.GET, userData, null, null, sshKeyPairs, + hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs, null, null, true, null, affinityGroupIdList, customParameters, null, null, null, null, true, overrideDiskOfferingId); } else { if (zone.isSecurityGroupEnabled()) { vm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, template, networkIds, null, owner, vmHostName,vmHostName, diskOfferingId, dataDiskSize, null, - hypervisorType, HTTPMethod.GET, userData, null, null, sshKeyPairs, + hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs, null, null, true, null, affinityGroupIdList, customParameters, null, null, null, null, true, overrideDiskOfferingId, null); } else { vm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, template, networkIds, owner, vmHostName, vmHostName, diskOfferingId, dataDiskSize, null, - hypervisorType, HTTPMethod.GET, userData, null, null, sshKeyPairs, + hypervisorType, HTTPMethod.GET, userData, userDataId, userDataDetails, sshKeyPairs, null, addrs, true, null, affinityGroupIdList, customParameters, null, null, null, null, true, null, overrideDiskOfferingId); } diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index 39f1e5d2d28..4f1396913cc 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -92,6 +92,10 @@ public interface UserVmManager extends UserVmService { void removeInstanceFromInstanceGroup(long vmId); + String finalizeUserData(String userData, Long userDataId, VirtualMachineTemplate template); + + String validateUserData(String userData, HTTPMethod httpmethod); + boolean isVMUsingLocalStorage(VMInstanceVO vm); boolean expunge(UserVmVO vm); diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index ecf60556db6..94ceb0de363 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -4769,7 +4769,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir } } - protected String validateUserData(String userData, HTTPMethod httpmethod) { + @Override + public String validateUserData(String userData, HTTPMethod httpmethod) { byte[] decodedUserData = null; if (userData != null) { @@ -5703,7 +5704,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir return userVm.getHypervisorType(); } - protected String finalizeUserData(String userData, Long userDataId, VirtualMachineTemplate template) { + @Override + public String finalizeUserData(String userData, Long userDataId, VirtualMachineTemplate template) { if (StringUtils.isEmpty(userData) && userDataId == null && (template == null || template.getUserDataId() == null)) { return null; } diff --git a/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java b/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java index f7cf7f58b2c..fff6fb2a950 100644 --- a/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java +++ b/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java @@ -32,6 +32,7 @@ import java.util.UUID; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse; +import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.DirectDownloadCertificateResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; import org.apache.cloudstack.api.response.UsageRecordResponse; @@ -52,17 +53,22 @@ import org.powermock.modules.junit4.PowerMockRunner; import com.cloud.domain.DomainVO; import com.cloud.network.as.AutoScaleVmGroup; import com.cloud.network.as.AutoScaleVmGroupVO; +import com.cloud.network.as.AutoScaleVmProfileVO; import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.LoadBalancerVO; import com.cloud.network.dao.NetworkServiceMapDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.storage.VMTemplateVO; import com.cloud.usage.UsageVO; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; import com.cloud.user.User; +import com.cloud.user.UserData; +import com.cloud.user.UserDataVO; import com.cloud.user.UserVO; +import com.cloud.user.dao.UserDataDao; import com.cloud.utils.net.Ip; import com.cloud.vm.NicSecondaryIp; @@ -86,12 +92,27 @@ public class ApiResponseHelperTest { @Mock AutoScaleVmGroupVmMapDao autoScaleVmGroupVmMapDaoMock; + @Mock + UserDataDao userDataDaoMock; + @Spy @InjectMocks ApiResponseHelper apiResponseHelper = new ApiResponseHelper(); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss ZZZ"); + static long zoneId = 1L; + static long domainId = 2L; + static long accountId = 3L; + static long serviceOfferingId = 4L; + static long templateId = 5L; + static String userdata = "userdata"; + static long userdataId = 6L; + static String userdataDetails = "userdataDetails"; + static String userdataNew = "userdataNew"; + + static long autoScaleUserId = 7L; + @Before public void injectMocks() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException { @@ -297,4 +318,55 @@ public class ApiResponseHelperTest { assertEquals("8080", response.getPublicPort()); assertEquals("8081", response.getPrivatePort()); } + + @Test + @PrepareForTest(ApiDBUtils.class) + public void testAutoScaleVmProfileResponse() { + AutoScaleVmProfileVO vmProfile = new AutoScaleVmProfileVO(zoneId, domainId, accountId, serviceOfferingId, templateId, null, null, userdata, null, autoScaleUserId); + vmProfile.setUserDataId(userdataId); + vmProfile.setUserDataDetails(userdataDetails); + + PowerMockito.mockStatic(ApiDBUtils.class); + when(ApiDBUtils.findAccountById(anyLong())).thenReturn(new AccountVO()); + when(ApiDBUtils.findDomainById(anyLong())).thenReturn(new DomainVO()); + + UserData.UserDataOverridePolicy templatePolicy = UserData.UserDataOverridePolicy.APPEND; + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + when(ApiDBUtils.findTemplateById(anyLong())).thenReturn(templateVO); + when(templateVO.getUserDataOverridePolicy()).thenReturn(templatePolicy); + + UserDataVO userDataVO = Mockito.mock(UserDataVO.class); + String userDataUuid = "userDataUuid"; + String userDataName = "userDataName"; + when(userDataDaoMock.findById(anyLong())).thenReturn(userDataVO); + when(userDataVO.getUuid()).thenReturn(userDataUuid); + when(userDataVO.getName()).thenReturn(userDataName); + + AutoScaleVmProfileResponse response = apiResponseHelper.createAutoScaleVmProfileResponse(vmProfile); + assertEquals(templatePolicy.toString(), response.getUserDataPolicy()); + assertEquals(userdata, response.getUserData()); + assertEquals(userDataUuid, response.getUserDataId()); + assertEquals(userDataName, response.getUserDataName()); + assertEquals(userdataDetails, response.getUserDataDetails()); + } + + @Test + @PrepareForTest(ApiDBUtils.class) + public void testAutoScaleVmProfileResponseWithoutUserData() { + AutoScaleVmProfileVO vmProfile = new AutoScaleVmProfileVO(zoneId, domainId, accountId, serviceOfferingId, templateId, null, null, null, null, autoScaleUserId); + + PowerMockito.mockStatic(ApiDBUtils.class); + when(ApiDBUtils.findAccountById(anyLong())).thenReturn(new AccountVO()); + when(ApiDBUtils.findDomainById(anyLong())).thenReturn(new DomainVO()); + + VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + when(ApiDBUtils.findTemplateById(anyLong())).thenReturn(templateVO); + + AutoScaleVmProfileResponse response = apiResponseHelper.createAutoScaleVmProfileResponse(vmProfile); + assertNull(response.getUserDataPolicy()); + assertNull(response.getUserData()); + assertNull(response.getUserDataId()); + assertNull(response.getUserDataName()); + assertNull(response.getUserDataDetails()); + } } diff --git a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java index e60ce86fc3f..08070faf92c 100644 --- a/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/as/AutoScaleManagerImplTest.java @@ -47,6 +47,7 @@ import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.admin.autoscale.CreateCounterCmd; import org.apache.cloudstack.api.command.user.autoscale.CreateAutoScalePolicyCmd; import org.apache.cloudstack.api.command.user.autoscale.CreateAutoScaleVmGroupCmd; @@ -340,6 +341,10 @@ public class AutoScaleManagerImplTest { private static final Long scaleDownCounterId = 38L; private static final Long nextVmSeq = 39L; private static final Long networkOfferingId = 40L; + private static final String userData = "VGVzdFVzZXJEYXRh"; //TestUserData + private static final Long userDataId = 41L; + private static final Map> userDataDetails = new HashMap<>(); + private static final String userDataFinal = "VGVzdFVzZXJEYXRhRmluYWw="; //TestUserDataFinal @Mock DataCenterVO zoneMock; @@ -404,6 +409,10 @@ public class AutoScaleManagerImplTest { Mockito.doNothing().when(accountManager).checkAccess(Mockito.any(Account.class), Mockito.isNull(), Mockito.anyBoolean(), Mockito.any()); when(asPolicyDao.persist(any(AutoScalePolicyVO.class))).thenReturn(asScaleUpPolicyMock); + + userDataDetails.put("0", new HashMap<>() {{ put("key1", "value1"); put("key2", "value2"); }}); + Mockito.doReturn(userDataFinal).when(userVmMgr).finalizeUserData(any(), any(), any()); + Mockito.doReturn(userDataFinal).when(userVmMgr).validateUserData(eq(userDataFinal), nullable(BaseCmd.HTTPMethod.class)); } @After @@ -748,10 +757,48 @@ public class AutoScaleManagerImplTest { ReflectionTestUtils.setField(cmd, "otherDeployParams", otherDeployParams); ReflectionTestUtils.setField(cmd, "counterParamList", counterParamList); + ReflectionTestUtils.setField(cmd, "userData", userData); + ReflectionTestUtils.setField(cmd, "userDataId", userDataId); + ReflectionTestUtils.setField(cmd, "userDataDetails", userDataDetails); + AutoScaleVmProfile vmProfile = autoScaleManagerImplSpy.createAutoScaleVmProfile(cmd); Assert.assertEquals(asVmProfileMock, vmProfile); Mockito.verify(autoScaleVmProfileDao).persist(Mockito.any()); + + Mockito.verify(userVmMgr).finalizeUserData(any(), any(), any()); + Mockito.verify(userVmMgr).validateUserData(eq(userDataFinal), nullable(BaseCmd.HTTPMethod.class)); + } + + @Test(expected = InvalidParameterValueException.class) + @PrepareForTest(ComponentContext.class) + public void testCreateAutoScaleVmProfileFail() { + when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(zoneMock); + when(entityManager.findById(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOfferingMock); + when(entityManager.findByIdIncludingRemoved(ServiceOffering.class, serviceOfferingId)).thenReturn(serviceOfferingMock); + when(entityManager.findById(VirtualMachineTemplate.class, templateId)).thenReturn(templateMock); + when(serviceOfferingMock.isDynamic()).thenReturn(false); + Mockito.doThrow(InvalidParameterValueException.class).when(userVmMgr).finalizeUserData(any(), any(), any()); + + DispatchChain dispatchChainMock = Mockito.mock(DispatchChain.class); + when(dispatchChainFactory.getStandardDispatchChain()).thenReturn(dispatchChainMock); + Mockito.doNothing().when(dispatchChainMock).dispatch(any()); + PowerMockito.mockStatic(ComponentContext.class); + when(ComponentContext.inject(DeployVMCmd.class)).thenReturn(Mockito.mock(DeployVMCmd.class)); + + CreateAutoScaleVmProfileCmd cmd = new CreateAutoScaleVmProfileCmd(); + + ReflectionTestUtils.setField(cmd, "zoneId", zoneId); + ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); + ReflectionTestUtils.setField(cmd, "templateId", templateId); + ReflectionTestUtils.setField(cmd, "expungeVmGracePeriod", expungeVmGracePeriod); + ReflectionTestUtils.setField(cmd, "otherDeployParams", otherDeployParams); + ReflectionTestUtils.setField(cmd, "counterParamList", counterParamList); + + ReflectionTestUtils.setField(cmd, "userData", userData); + ReflectionTestUtils.setField(cmd, "userDataId", userDataId); + + AutoScaleVmProfile vmProfile = autoScaleManagerImplSpy.createAutoScaleVmProfile(cmd); } @Test @@ -774,10 +821,17 @@ public class AutoScaleManagerImplTest { ReflectionTestUtils.setField(cmd, "serviceOfferingId", serviceOfferingId); ReflectionTestUtils.setField(cmd, "templateId", templateId); + ReflectionTestUtils.setField(cmd, "userData", userData); + ReflectionTestUtils.setField(cmd, "userDataId", userDataId); + ReflectionTestUtils.setField(cmd, "userDataDetails", userDataDetails); + AutoScaleVmProfile vmProfile = autoScaleManagerImplSpy.updateAutoScaleVmProfile(cmd); Assert.assertEquals(asVmProfileMock, vmProfile); Mockito.verify(autoScaleVmProfileDao).persist(Mockito.any()); + + Mockito.verify(userVmMgr).finalizeUserData(any(), any(), any()); + Mockito.verify(userVmMgr).validateUserData(eq(userDataFinal), nullable(BaseCmd.HTTPMethod.class)); } @Test @@ -1208,6 +1262,9 @@ public class AutoScaleManagerImplTest { when(asVmProfileMock.getAccountId()).thenReturn(accountId); when(asVmProfileMock.getZoneId()).thenReturn(zoneId); when(asVmProfileMock.getOtherDeployParams()).thenReturn(""); + when(asVmProfileMock.getUserData()).thenReturn(userData); + when(asVmProfileMock.getUserDataId()).thenReturn(userDataId); + when(asVmProfileMock.getUserDataDetails()).thenReturn(userDataDetails.toString()); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(zoneMock); @@ -1224,7 +1281,7 @@ public class AutoScaleManagerImplTest { when(userVmMock.getId()).thenReturn(virtualMachineId); when(zoneMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Basic); when(userVmService.createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any())).thenReturn(userVmMock); long result = autoScaleManagerImplSpy.createNewVM(asVmGroupMock); @@ -1235,7 +1292,7 @@ public class AutoScaleManagerImplTest { "-" + asVmGroupMock.getNextVmSeq() + "-[a-z]{6}"; Mockito.verify(userVmService).createBasicSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), matches(vmHostNamePattern), matches(vmHostNamePattern), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any()); Mockito.verify(asVmGroupMock).setNextVmSeq(nextVmSeq + 1); } @@ -1253,6 +1310,9 @@ public class AutoScaleManagerImplTest { when(asVmProfileMock.getAccountId()).thenReturn(accountId); when(asVmProfileMock.getZoneId()).thenReturn(zoneId); when(asVmProfileMock.getOtherDeployParams()).thenReturn(""); + when(asVmProfileMock.getUserData()).thenReturn(userData); + when(asVmProfileMock.getUserDataId()).thenReturn(userDataId); + when(asVmProfileMock.getUserDataDetails()).thenReturn(userDataDetails.toString()); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(zoneMock); @@ -1270,7 +1330,7 @@ public class AutoScaleManagerImplTest { when(zoneMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); when(zoneMock.isSecurityGroupEnabled()).thenReturn(true); when(userVmService.createAdvancedSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any())).thenReturn(userVmMock); long result = autoScaleManagerImplSpy.createNewVM(asVmGroupMock); @@ -1281,7 +1341,7 @@ public class AutoScaleManagerImplTest { "-" + asVmGroupMock.getNextVmSeq() + "-[a-z]{6}"; Mockito.verify(userVmService).createAdvancedSecurityGroupVirtualMachine(any(), any(), any(), any(), any(), any(), matches(vmHostNamePattern), matches(vmHostNamePattern), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), + any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any()); Mockito.verify(asVmGroupMock).setNextVmSeq(nextVmSeq + 2); } @@ -1299,6 +1359,9 @@ public class AutoScaleManagerImplTest { when(asVmProfileMock.getAccountId()).thenReturn(accountId); when(asVmProfileMock.getZoneId()).thenReturn(zoneId); when(asVmProfileMock.getOtherDeployParams()).thenReturn(""); + when(asVmProfileMock.getUserData()).thenReturn(userData); + when(asVmProfileMock.getUserDataId()).thenReturn(userDataId); + when(asVmProfileMock.getUserDataDetails()).thenReturn(userDataDetails.toString()); when(accountService.getActiveAccountById(accountId)).thenReturn(account); when(entityManager.findById(DataCenter.class, zoneId)).thenReturn(zoneMock); @@ -1316,7 +1379,7 @@ public class AutoScaleManagerImplTest { when(zoneMock.getNetworkType()).thenReturn(DataCenter.NetworkType.Advanced); when(zoneMock.isSecurityGroupEnabled()).thenReturn(false); when(userVmService.createAdvancedVirtualMachine(any(), any(), any(), any(), any(), any(), any(), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any())).thenReturn(userVmMock); long result = autoScaleManagerImplSpy.createNewVM(asVmGroupMock); @@ -1327,7 +1390,7 @@ public class AutoScaleManagerImplTest { "-" + asVmGroupMock.getNextVmSeq() + "-[a-z]{6}"; Mockito.verify(userVmService).createAdvancedVirtualMachine(any(), any(), any(), any(), any(), matches(vmHostNamePattern), matches(vmHostNamePattern), - any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any(), any(), + any(), any(), any(), any(), any(), eq(userData), eq(userDataId), eq(userDataDetails.toString()), any(), any(), any(), eq(true), any(), any(), any(), any(), any(), any(), any(), eq(true), any(), any()); Mockito.verify(asVmGroupMock).setNextVmSeq(nextVmSeq + 3); } diff --git a/test/integration/smoke/test_vm_autoscaling.py b/test/integration/smoke/test_vm_autoscaling.py index d9fa7e23b79..316f94fd5fd 100644 --- a/test/integration/smoke/test_vm_autoscaling.py +++ b/test/integration/smoke/test_vm_autoscaling.py @@ -38,6 +38,7 @@ from marvin.lib.base import (Account, Domain, Project, ServiceOffering, + Template, VirtualMachine, Volume, Zone, @@ -47,6 +48,7 @@ from marvin.lib.base import (Account, LoadBalancerRule, VPC, VpcOffering, + UserData, SSHKeyPair) from marvin.lib.common import (get_domain, @@ -198,6 +200,37 @@ class TestVmAutoScaling(cloudstackTestCase): name="keypair2" ) + # 8-2. Register userdata + cls.apiUserdata = UserData.register( + cls.apiclient, + name="ApiUserdata", + userdata="QVBJdXNlcmRhdGE=", #APIuserdata + account=cls.regular_user.name, + domainid=cls.regular_user.domainid + ) + + # 8-3. Register userdata for template + cls.templateUserdata = UserData.register( + cls.apiclient, + name="TemplateUserdata", + userdata="IyMgdGVtcGxhdGU6IGppbmphCiNjbG91ZC1jb25maWcKcnVuY21kOgogICAgLSBlY2hvICdrZXkge3sgZHMubWV0YV9kYXRhLmtleTEgfX0nID4+IC9yb290L2luc3RhbmNlX21ldGFkYXRhCgo=", + # ## template: jinja + # #cloud-config + # runcmd: + # - echo 'key {{ ds.meta_data.key1 }}' >> /root/instance_metadata + # + account=cls.regular_user.name, + domainid=cls.regular_user.domainid + ) + + # 8-3. Link userdata to template + cls.template = Template.linkUserDataToTemplate( + cls.apiclient, + templateid=cls.template.id, + userdataid=cls.templateUserdata.userdata.id, + userdatapolicy="append" + ) + # 9. Get counters for cpu and memory counters = Autoscale.listCounters( cls.regular_user_apiclient, @@ -294,6 +327,7 @@ class TestVmAutoScaling(cloudstackTestCase): serviceofferingid=cls.service_offering.id, zoneid=cls.zone.id, templateid=cls.template.id, + userdata="VGVzdFVzZXJEYXRh", #TestUserData expungevmgraceperiod=DEFAULT_EXPUNGE_VM_GRACE_PERIOD, otherdeployparams=cls.otherdeployparams ) @@ -349,6 +383,10 @@ class TestVmAutoScaling(cloudstackTestCase): @classmethod def tearDownClass(cls): + cls.template = Template.linkUserDataToTemplate( + cls.apiclient, + templateid=cls.template.id + ) Configurations.update(cls.apiclient, CONFIG_NAME_DISK_CONTROLLER, cls.initial_vmware_root_disk_controller) @@ -390,6 +428,7 @@ class TestVmAutoScaling(cloudstackTestCase): self.regular_user_apiclient, autoscalevmgroupid=autoscalevmgroupid, projectid=projectid, + userdata=True, listall=True ) self.assertEqual( @@ -505,6 +544,31 @@ class TestVmAutoScaling(cloudstackTestCase): else: self.assertEquals(affinitygroupids, '') + userdata = None + userdatadetails = None + userdataid = None + if vm.userdata: + userdata = vm.userdata + if vm.userdatadetails: + userdatadetails = vm.userdatadetails + if vm.userdataid: + userdataid = vm.userdataid + + if vmprofile.userdataid: + self.assertEquals(userdataid, vmprofile.userdataid) + else: + self.assertIsNone(userdataid) + + if vmprofile.userdatadetails: + self.assertEquals(userdatadetails, vmprofile.userdatadetails) + else: + self.assertIsNone(userdatadetails) + + if vmprofile.userdata: + self.assertEquals(userdata, vmprofile.userdata) + else: + self.assertIsNone(userdata) + def wait_for_vm_start(self, vm=None, project_id=None): """ Wait until vm is Running """ def check_user_vm_state(): @@ -512,6 +576,7 @@ class TestVmAutoScaling(cloudstackTestCase): self.apiclient, id=vm.id, projectid=project_id, + userdata=True, listall=True ) if isinstance(vms, list): @@ -576,6 +641,8 @@ class TestVmAutoScaling(cloudstackTestCase): Autoscale.updateAutoscaleVMProfile( self.regular_user_apiclient, id = self.autoscaling_vmprofile.id, + userdataid=self.apiUserdata.userdata.id, + userdatadetails=[{"key1": "value2"}], serviceofferingid = self.service_offering_new.id, expungevmgraceperiod = DEFAULT_EXPUNGE_VM_GRACE_PERIOD + 1, otherdeployparams = otherdeployparams_new @@ -712,6 +779,7 @@ class TestVmAutoScaling(cloudstackTestCase): vms = VirtualMachine.list( self.regular_user_apiclient, autoscalevmgroupid=self.autoscaling_vmgroup.id, + userdata=True, listall=True ) self.assertEqual( @@ -889,6 +957,7 @@ class TestVmAutoScaling(cloudstackTestCase): self.regular_user_apiclient, autoscalevmgroupid=autoscaling_vmgroup_project.id, projectid=project.id, + userdata=True, listall=True ) self.assertEqual( diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index f93cbd233f7..dcdd0bf97c1 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1625,6 +1625,7 @@ "label.reset.config.value": "Reset to default value", "label.reset.ssh.key.pair": "Reset SSH key pair", "label.reset.to.default": "Reset to default", +"label.reset.userdata.on.autoscale.vm.group": "Reset Userdata on AutoScale VM Group", "label.reset.userdata.on.vm": "Reset Userdata on VM", "label.reset.vpn.connection": "Reset VPN connection", "label.resource": "Resource", diff --git a/ui/src/views/compute/AutoScaleVmProfile.vue b/ui/src/views/compute/AutoScaleVmProfile.vue index 45947753b06..12081dfad72 100644 --- a/ui/src/views/compute/AutoScaleVmProfile.vue +++ b/ui/src/views/compute/AutoScaleVmProfile.vue @@ -69,6 +69,47 @@ {{ getServiceOfferingName(serviceofferingid) }}
+
+
+
+ +
+ {{ userdataid }} +
+
+
+
+
+ +
+ {{ userdataname }} +
+
+
+
+
+ +
+ {{ userdatadetails }} +
+
+
+
+
+ +
+ {{ userdatapolicy }} +
+
+
+
+
+ +
+ + +
+
@@ -76,6 +117,12 @@ {{ $t('label.edit.autoscale.vmprofile') }}
+
+ + + {{ $t('label.reset.userdata.on.autoscale.vm.group') }} + +
@@ -224,20 +271,26 @@
-
-
-
- -
- - -
-
{{ $t('label.cancel') }} {{ $t('label.ok') }}
+ + + + @@ -247,10 +300,12 @@ import { isAdmin, isAdminOrDomainAdmin } from '@/role' import Status from '@/components/widgets/Status' import TooltipButton from '@/components/widgets/TooltipButton' import TooltipLabel from '@/components/widgets/TooltipLabel' +import ResetUserData from '@views/compute/ResetUserData' export default { name: 'conditionsTab', components: { + ResetUserData, Status, TooltipButton, TooltipLabel @@ -266,12 +321,17 @@ export default { filterColumns: ['Action'], loading: true, editProfileModalVisible: false, + showUpdateUserDataForm: false, profileid: null, autoscaleuserid: null, expungevmgraceperiod: null, templateid: null, serviceofferingid: null, userdata: null, + userdataid: null, + userdataname: null, + userdatadetails: null, + userdatapolicy: null, usersList: [], templatesList: [], serviceOfferingsList: [], @@ -384,6 +444,11 @@ export default { this.serviceofferingid = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.serviceofferingid this.templateid = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.templateid this.userdata = this.decodeUserData(decodeURIComponent(response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.userdata || '')) + this.userdataid = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.userdataid + this.userdataname = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.userdataname + this.userdatadetails = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.userdatadetails + this.userdatapolicy = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.userdatapolicy + const counterparam = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.counterparam || {} const otherdeployparams = response.listautoscalevmprofilesresponse?.autoscalevmprofile?.[0]?.otherdeployparams || {} this.finalizeParams(counterparam, otherdeployparams) @@ -518,13 +583,10 @@ export default { if (this.autoscaleuserid) { params.autoscaleuserid = this.autoscaleuserid } - if (this.userdata && this.userdata.length > 0) { - params.userdata = this.$toBase64AndURIEncoded(this.userdata) - } - const httpMethod = params.userdata ? 'POST' : 'GET' - const args = httpMethod === 'POST' ? {} : params - const data = httpMethod === 'POST' ? params : {} + const httpMethod = 'GET' + const args = params + const data = {} api('updateAutoScaleVmProfile', args, httpMethod, data).then(response => { this.$pollJob({ diff --git a/ui/src/views/compute/CreateAutoScaleVmGroup.vue b/ui/src/views/compute/CreateAutoScaleVmGroup.vue index b764f8185e2..81da8a9b6d7 100644 --- a/ui/src/views/compute/CreateAutoScaleVmGroup.vue +++ b/ui/src/views/compute/CreateAutoScaleVmGroup.vue @@ -407,7 +407,7 @@ @@ -760,13 +760,100 @@ @select-affinity-group-item="($event) => updateAffinityGroups($event)" @handle-search-filter="($event) => handleSearchFilter('affinityGroups', $event)"/> - + - - + +
+ + Userdata "{{ $t(this.template.userdataname) }}" is linked with template "{{ $t(this.template.name) }}" with override policy "{{ $t(this.template.userdatapolicy) }}" +

+
+ + Enter the values for the variables in userdata + + + + + + +
+


+
+ + {{ $t('label.userdata.do.override') }} + + + + {{ $t('label.userdata.do.append') }} + + + + + +
+ @@ -952,6 +1039,7 @@ import NetworkSelection from '@views/compute/wizard/NetworkSelection' import NetworkConfiguration from '@views/compute/wizard/NetworkConfiguration' import LoadBalancerSelection from '@views/compute/wizard/LoadBalancerSelection' import SshKeyPairSelection from '@views/compute/wizard/SshKeyPairSelection' +import UserDataSelection from '@views/compute/wizard/UserDataSelection' import SecurityGroupSelection from '@views/compute/wizard/SecurityGroupSelection' import TooltipLabel from '@/components/widgets/TooltipLabel' import InstanceNicsNetworkSelectListView from '@/components/view/InstanceNicsNetworkSelectListView.vue' @@ -964,6 +1052,7 @@ export default { name: 'Wizard', components: { SshKeyPairSelection, + UserDataSelection, NetworkConfiguration, NetworkSelection, LoadBalancerSelection, @@ -1010,6 +1099,10 @@ export default { zoneSelected: false, dynamicscalingenabled: true, templateKey: 0, + showRegisteredUserdata: true, + doUserdataOverride: false, + doUserdataAppend: false, + userdataDefaultOverridePolicy: 'ALLOWOVERRIDE', vm: { name: null, zoneid: null, @@ -1034,6 +1127,7 @@ export default { affinityGroups: [], networks: [], sshKeyPairs: [], + UserDatas: [], loadbalancers: [] }, rowCount: {}, @@ -1045,6 +1139,7 @@ export default { affinityGroups: false, networks: false, sshKeyPairs: false, + userDatas: false, loadbalancers: false, zones: false }, @@ -1123,6 +1218,32 @@ export default { zone: {}, sshKeyPairs: [], sshKeyPair: {}, + userData: {}, + userDataParams: [], + userDataParamCols: [ + { + title: this.$t('label.key'), + dataIndex: 'key' + }, + { + title: this.$t('label.value'), + dataIndex: 'value', + slots: { customRender: 'value' } + } + ], + userDataValues: {}, + templateUserDataCols: [ + { + title: this.$t('label.userdata'), + dataIndex: 'userdata' + }, + { + title: this.$t('label.userdatapolicy'), + dataIndex: 'userdataoverridepolicy' + } + ], + templateUserDataParams: [], + templateUserDataValues: {}, overrideDiskOffering: {}, templateFilter: [ 'featured', @@ -1134,6 +1255,7 @@ export default { defaultNetworkId: '', dataNetworkCreated: [], tabKey: 'templateid', + userdataTabKey: 'userdataregistered', dataPreFill: {}, showDetails: false, showRootDiskSizeChanger: false, @@ -1224,6 +1346,15 @@ export default { listall: false } }, + userDatas: { + list: 'listUserData', + options: { + page: 1, + pageSize: 10, + keyword: undefined, + listall: false + } + }, networks: { list: 'listNetworks', options: { @@ -1285,12 +1416,27 @@ export default { }] return tabList }, + userdataTabList () { + let tabList = [] + tabList = [{ + key: 'userdataregistered', + tab: this.$t('label.userdata.registered') + }, + { + key: 'userdatatext', + tab: this.$t('label.userdata.text') + }] + return tabList + }, showSecurityGroupSection () { return (this.networks.length > 0 && this.zone.securitygroupsenabled) || (this.zone && this.zone.networktype === 'Basic') }, isUserAllowedToListSshKeys () { return Boolean('listSSHKeyPairs' in this.$store.getters.apis) }, + isUserAllowedToListUserDatas () { + return Boolean('listUserData' in this.$store.getters.apis) + }, dynamicScalingVmConfigValue () { return this.options.dynamicScalingVmConfig?.[0]?.value === 'true' }, @@ -1421,6 +1567,8 @@ export default { template (oldValue, newValue) { if (oldValue && newValue && oldValue.id !== newValue.id) { this.dynamicscalingenabled = this.isDynamicallyScalable() + this.doUserdataOverride = false + this.doUserdataAppend = false } }, beforeCreate () { @@ -1853,6 +2001,8 @@ export default { if (template) { var size = template.size / (1024 * 1024 * 1024) || 0 // bytes to GB this.dataPreFill.minrootdisksize = Math.ceil(size) + this.updateTemplateLinkedUserData(this.template.userdataid) + this.userdataDefaultOverridePolicy = this.template.userdatapolicy } } else if (['cpuspeed', 'cpunumber', 'memory'].includes(name)) { this.vm[name] = value @@ -2017,6 +2167,56 @@ export default { this.form.keypairs = names this.sshKeyPairs = names.map((sshKeyPair) => { return sshKeyPair.name }) }, + updateUserData (id) { + if (id === '0') { + this.form.userdataid = undefined + return + } + this.form.userdataid = id + this.userDataParams = [] + api('listUserData', { id: id }).then(json => { + const resp = json?.listuserdataresponse?.userdata || [] + if (resp) { + var params = resp[0].params + if (params) { + var dataParams = params.split(',') + } + var that = this + dataParams.forEach(function (val, index) { + that.userDataParams.push({ + id: index, + key: val + }) + }) + } + }) + }, + updateTemplateLinkedUserData (id) { + if (id === '0') { + return + } + this.templateUserDataParams = [] + + api('listUserData', { id: id }).then(json => { + const resp = json?.listuserdataresponse?.userdata || [] + if (resp) { + var params = resp[0].params + if (params) { + var dataParams = params.split(',') + } + var that = this + that.templateUserDataParams = [] + if (dataParams) { + dataParams.forEach(function (val, index) { + that.templateUserDataParams.push({ + id: index, + key: val + }) + }) + } + } + }) + }, updateAffinityGroups (ids) { this.form.affinitygroupids = ids }, @@ -2035,16 +2235,20 @@ export default { }, 1000) }) }, - createVmProfile (createVmGroupData) { + createVmProfile (createVmGroupData, createVmGroupUserDataDetails) { this.addStep('message.creating.autoscale.vmprofile', 'createVmProfile') return new Promise((resolve, reject) => { const params = { - expungevmgraceperiod: createVmGroupData.expungevmgraceperiod, - serviceofferingid: createVmGroupData.serviceofferingid, - templateid: createVmGroupData.templateid, - userdata: createVmGroupData.userdata, - zoneid: createVmGroupData.zoneid + ...createVmGroupUserDataDetails, + ...{ + expungevmgraceperiod: createVmGroupData.expungevmgraceperiod, + serviceofferingid: createVmGroupData.serviceofferingid, + templateid: createVmGroupData.templateid, + userdata: createVmGroupData.userdata, + userdataid: createVmGroupData.userdataid, + zoneid: createVmGroupData.zoneid + } } if (createVmGroupData.autoscaleuserid) { params.autoscaleuserid = createVmGroupData.autoscaleuserid @@ -2425,9 +2629,13 @@ export default { // advanced settings createVmGroupData.keypairs = this.sshKeyPairs.join(',') createVmGroupData.affinitygroupids = (values.affinitygroupids || []).join(',') - if (values.userdata && values.userdata.length > 0) { + const isUserdataAllowed = !this.userdataDefaultOverridePolicy || (this.userdataDefaultOverridePolicy === 'ALLOWOVERRIDE' && this.doUserdataOverride) || (this.userdataDefaultOverridePolicy === 'APPEND' && this.doUserdataAppend) + if (isUserdataAllowed && values.userdata && values.userdata.length > 0) { createVmGroupData.userdata = this.$toBase64AndURIEncoded(values.userdata) } + if (isUserdataAllowed) { + createVmGroupData.userdataid = values.userdataid + } // vm profile details createVmGroupData.autoscaleuserid = values.autoscaleuserid @@ -2436,11 +2644,26 @@ export default { createVmGroupData = Object.fromEntries( Object.entries(createVmGroupData).filter(([key, value]) => value !== undefined)) + const createVmGroupUserDataDetails = {} + var idx = 0 + if (this.templateUserDataValues) { + for (const [key, value] of Object.entries(this.templateUserDataValues)) { + createVmGroupUserDataDetails['userdatadetails[' + idx + '].' + `${key}`] = value + idx++ + } + } + if (isUserdataAllowed && this.userDataValues) { + for (const [key, value] of Object.entries(this.userDataValues)) { + createVmGroupUserDataDetails['userdatadetails[' + idx + '].' + `${key}`] = value + idx++ + } + } + this.processStatusModalVisible = true this.processStatus = null // create autoscale vm profile - const vmprofile = await this.createVmProfile(createVmGroupData) + const vmprofile = await this.createVmProfile(createVmGroupData, createVmGroupUserDataDetails) // create scaleup conditions and policy const scaleUpPolicyIds = [] @@ -2547,7 +2770,7 @@ export default { return new Promise((resolve) => { this.loading.zones = true const param = this.params.zones - const args = { listall: true, showicon: true } + const args = { showicon: true } if (zoneId) args.id = zoneId api(param.list, args).then(json => { const zoneResponse = (json.listzonesresponse.zone || []).filter(item => item.securitygroupsenabled === false) @@ -2579,7 +2802,7 @@ export default { param.loading = true param.opts = [] const options = param.options || {} - if (!('listall' in options)) { + if (!('listall' in options) && !['zones', 'pods', 'clusters', 'hosts', 'dynamicScalingVmConfig', 'hypervisors'].includes(name)) { options.listall = true } api(param.list, options).then((response) => { @@ -2703,6 +2926,10 @@ export default { this.params[name].options = { ...this.params[name].options, ...options } this.fetchOptions(this.params[name], name) }, + onUserdataTabChange (key, type) { + this[type] = key + this.userDataParams = [] + }, fetchTemplateNics (template) { var nics = [] this.nicToNetworkSelection = [] diff --git a/ui/src/views/compute/ResetUserData.vue b/ui/src/views/compute/ResetUserData.vue index 46561f15c1a..b9753f4bc19 100644 --- a/ui/src/views/compute/ResetUserData.vue +++ b/ui/src/views/compute/ResetUserData.vue @@ -335,7 +335,6 @@ export default { this.loadingData = true console.log(values) const params = { - id: this.resource.id } if (values.userdata && values.userdata.length > 0) { params.userdata = this.$toBase64AndURIEncoded(values.userdata) @@ -356,7 +355,14 @@ export default { idx++ } } - api('resetUserDataForVirtualMachine', params).then(json => { + params.id = this.resource.resetUserDataResourceId ? this.resource.resetUserDataResourceId : this.resource.id + + const resetUserDataApiName = this.resource.resetUserDataApiName ? this.resource.resetUserDataApiName : 'resetUserDataForVirtualMachine' + const httpMethod = params.userdata ? 'POST' : 'GET' + const args = httpMethod === 'POST' ? {} : params + const data = httpMethod === 'POST' ? params : {} + + api(resetUserDataApiName, args, httpMethod, data).then(json => { this.$message.success({ content: `${this.$t('label.action.userdata.reset')} - ${this.$t('label.success')}`, duration: 2 From 5097d2aa8bd8f1c360b6202df1b5ffa531740316 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 22 Aug 2023 11:15:30 +0200 Subject: [PATCH 24/31] ui: fix creating zone with vxlan if Guest physical network is not the last (#7801) This fixes #7790 --- ui/src/views/infra/zone/ZoneWizardLaunchZone.vue | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue index bf43e4bd5c4..ce8769801cf 100644 --- a/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue +++ b/ui/src/views/infra/zone/ZoneWizardLaunchZone.vue @@ -483,6 +483,9 @@ export default { } else { this.stepData.physicalNetworkReturned = this.stepData.physicalNetworkItem['createPhysicalNetwork' + index] } + if (physicalNetwork.traffics.findIndex(traffic => traffic.type === 'guest') > -1) { + this.stepData.guestPhysicalNetworkId = physicalNetworkReturned.id + } } catch (e) { this.messageError = e this.processStatus = STATUS_FAILED @@ -1181,7 +1184,7 @@ export default { } const updateParams = {} - updateParams.id = this.stepData.physicalNetworkReturned.id + updateParams.id = this.stepData.guestPhysicalNetworkId updateParams.vlan = vlan try { From db0e7a65afbe839a2ea452f85200176d81cc4779 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 22 Aug 2023 11:22:20 +0200 Subject: [PATCH 25/31] server: check hostId when attach disk to a Stopped vm with local storage (#7886) This fixes #7834 --- .../orchestration/VolumeOrchestrator.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index f7c8c9c70bf..52bcf5033af 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -123,6 +123,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StorageUtil; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.Volume; @@ -131,6 +132,7 @@ import com.cloud.storage.VolumeApiService; import com.cloud.storage.VolumeDetailVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDetailsDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; @@ -243,6 +245,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati VolumeApiService _volumeApiService; @Inject PassphraseDao passphraseDao; + @Inject + StoragePoolHostDao storagePoolHostDao; @Inject protected SnapshotHelper snapshotHelper; @@ -656,7 +660,8 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati } @DB - public VolumeInfo createVolume(VolumeInfo volumeInfo, VirtualMachine vm, VirtualMachineTemplate template, DataCenter dc, Pod pod, Long clusterId, ServiceOffering offering, DiskOffering diskOffering, + public VolumeInfo createVolume(VolumeInfo volumeInfo, VirtualMachine vm, VirtualMachineTemplate template, DataCenter dc, Pod pod, Long clusterId, + Long hostId, ServiceOffering offering, DiskOffering diskOffering, List avoids, long size, HypervisorType hyperType) { // update the volume's hv_ss_reserve (hypervisor snapshot reserve) from a disk offering (used for managed storage) volumeInfo = volService.updateHypervisorSnapshotReserveForVolume(diskOffering, volumeInfo.getId(), hyperType); @@ -691,7 +696,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati final HashSet avoidPools = new HashSet(avoids); - pool = findStoragePool(dskCh, dc, pod, clusterId, vm.getHostId(), vm, avoidPools); + pool = findStoragePool(dskCh, dc, pod, clusterId, hostId, vm, avoidPools); if (pool == null) { String msg = String.format("Unable to find suitable primary storage when creating volume [%s].", volumeToString); s_logger.error(msg); @@ -1122,10 +1127,17 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati if (s_logger.isTraceEnabled()) { s_logger.trace(String.format("storage-pool %s/%s is associated with cluster %d",storagePool.getName(), storagePool.getUuid(), clusterId)); } + Long hostId = vm.getHostId(); + if (hostId == null && storagePool.isLocal()) { + List poolHosts = storagePoolHostDao.listByPoolId(storagePool.getId()); + if (poolHosts.size() > 0) { + hostId = poolHosts.get(0).getHostId(); + } + } VolumeInfo vol = null; if (volumeInfo.getState() == Volume.State.Allocated) { - vol = createVolume(volumeInfo, vm, rootDiskTmplt, dcVO, pod, clusterId, svo, diskVO, new ArrayList(), volumeInfo.getSize(), rootDiskHyperType); + vol = createVolume(volumeInfo, vm, rootDiskTmplt, dcVO, pod, clusterId, hostId, svo, diskVO, new ArrayList(), volumeInfo.getSize(), rootDiskHyperType); } else if (volumeInfo.getState() == Volume.State.Uploaded) { vol = copyVolume(storagePool, volumeInfo, vm, rootDiskTmplt, dcVO, pod, diskVO, svo, rootDiskHyperType); if (vol != null) { From 2ef159eeb84ee2a69a3bee774f5fa2c64cbf8c78 Mon Sep 17 00:00:00 2001 From: Daan Hoogland Date: Tue, 22 Aug 2023 13:44:21 +0200 Subject: [PATCH 26/31] merge issues --- server/src/main/java/com/cloud/vm/UserVmManager.java | 2 ++ server/src/main/java/com/cloud/vm/UserVmManagerImpl.java | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/server/src/main/java/com/cloud/vm/UserVmManager.java b/server/src/main/java/com/cloud/vm/UserVmManager.java index c2d360abb13..4f1396913cc 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManager.java +++ b/server/src/main/java/com/cloud/vm/UserVmManager.java @@ -58,6 +58,8 @@ public interface UserVmManager extends UserVmService { "Destroys the VM's root volume when the VM is destroyed.", true, ConfigKey.Scope.Domain); + static final int MAX_USER_DATA_LENGTH_BYTES = 2048; + public static final String CKS_NODE = "cksnode"; /** diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 50976aae9b7..1e92b14d82e 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.vm; +import static com.cloud.configuration.ConfigurationManagerImpl.VM_USERDATA_MAX_LENGTH; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import java.io.IOException; @@ -126,6 +127,7 @@ import org.apache.cloudstack.userdata.UserDataManager; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.security.ParserUtils; import org.apache.cloudstack.vm.schedule.VMScheduleManager; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.math.NumberUtils; @@ -601,6 +603,10 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir protected static long ROOT_DEVICE_ID = 0; + private static final int MAX_HTTP_GET_LENGTH = 2 * MAX_USER_DATA_LENGTH_BYTES; + private static final int NUM_OF_2K_BLOCKS = 512; + private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS * MAX_USER_DATA_LENGTH_BYTES; + @Inject private OrchestrationService _orchSrvc; From 0e9a19f24db7b7585f1b952b403b3f6e619cb53f Mon Sep 17 00:00:00 2001 From: Rohit Yadav Date: Tue, 22 Aug 2023 17:50:45 +0530 Subject: [PATCH 27/31] ui: Fix project theme on reload or refresh (#7893) * css fixes from main Signed-off-by: Rohit Yadav * component: when in project, toggle the theme also Signed-off-by: Rohit Yadav --------- Signed-off-by: Rohit Yadav --- ui/src/components/header/ProjectMenu.vue | 1 + ui/src/views/compute/CreateAutoScaleVmGroup.vue | 4 ++-- ui/src/views/compute/DeployVM.vue | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ui/src/components/header/ProjectMenu.vue b/ui/src/components/header/ProjectMenu.vue index 722bc2877d6..95a12ffeb1d 100644 --- a/ui/src/components/header/ProjectMenu.vue +++ b/ui/src/components/header/ProjectMenu.vue @@ -78,6 +78,7 @@ export default { let projectIndex = 0 if (this.$store.getters?.project?.id) { projectIndex = this.projects.findIndex(project => project.id === this.$store.getters.project.id) + this.$store.dispatch('ToggleTheme', projectIndex === undefined ? 'light' : 'dark') } return projectIndex diff --git a/ui/src/views/compute/CreateAutoScaleVmGroup.vue b/ui/src/views/compute/CreateAutoScaleVmGroup.vue index 81da8a9b6d7..b9dce0ed500 100644 --- a/ui/src/views/compute/CreateAutoScaleVmGroup.vue +++ b/ui/src/views/compute/CreateAutoScaleVmGroup.vue @@ -34,7 +34,7 @@
{{ $t('message.select.a.zone') }}
-
+
{{ $t('message.select.a.zone') }}
-
+
Date: Wed, 23 Aug 2023 09:44:34 +0200 Subject: [PATCH 28/31] engine/schema: fix duplicated guest OSes in 4.18.0.0 (#7799) Co-authored-by: Daan Hoogland --- .../main/java/com/cloud/storage/GuestOS.java | 2 + .../command/admin/guest/AddGuestOsCmd.java | 8 +- .../command/admin/guest/UpdateGuestOsCmd.java | 11 +- .../command/user/guest/ListGuestOsCmd.java | 14 ++- .../api/response/GuestOSResponse.java | 12 ++ .../java/com/cloud/storage/GuestOSVO.java | 14 +++ .../com/cloud/storage/dao/GuestOSDao.java | 13 ++- .../com/cloud/storage/dao/GuestOSDaoImpl.java | 78 ++++++++++++- .../storage/dao/GuestOSHypervisorDao.java | 8 +- .../storage/dao/GuestOSHypervisorDaoImpl.java | 8 +- .../java/com/cloud/upgrade/GuestOsMapper.java | 110 ++++++++++++++++-- .../upgrade/dao/Upgrade41800to41810.java | 12 ++ .../META-INF/db/schema-41720to41800.sql | 4 +- .../META-INF/db/schema-41800to41810.sql | 4 + .../com/cloud/hypervisor/guru/VMwareGuru.java | 4 +- .../main/java/com/cloud/api/ApiDBUtils.java | 2 +- .../java/com/cloud/api/ApiResponseHelper.java | 1 + .../cloud/server/ManagementServerImpl.java | 49 +++----- .../vm/UnmanagedVMsManagerImpl.java | 2 +- 19 files changed, 295 insertions(+), 61 deletions(-) diff --git a/api/src/main/java/com/cloud/storage/GuestOS.java b/api/src/main/java/com/cloud/storage/GuestOS.java index 371260bec64..90b904758a9 100644 --- a/api/src/main/java/com/cloud/storage/GuestOS.java +++ b/api/src/main/java/com/cloud/storage/GuestOS.java @@ -34,4 +34,6 @@ public interface GuestOS extends InternalIdentity, Identity { Date getRemoved(); boolean getIsUserDefined(); + + boolean getForDisplay(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCmd.java index 2b95c209bb8..b0d59d49362 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCmd.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.GuestOSResponse; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.context.CallContext; import com.cloud.event.EventTypes; @@ -60,6 +61,8 @@ public class AddGuestOsCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, required = false, description = "Map of (key/value pairs)") private Map details; + @Parameter(name="forDisplay", type=CommandType.BOOLEAN, description="whether this guest OS is available for end users", authorized = {RoleType.Admin}) + private Boolean display; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -78,7 +81,7 @@ public class AddGuestOsCmd extends BaseAsyncCreateCmd { } public Map getDetails() { - Map detailsMap = new HashMap(); + Map detailsMap = new HashMap<>(); if (!details.isEmpty()) { Collection servicesCollection = details.values(); Iterator iter = servicesCollection.iterator(); @@ -92,6 +95,9 @@ public class AddGuestOsCmd extends BaseAsyncCreateCmd { return detailsMap; } + public Boolean getForDisplay() { + return display; + } ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java index 1168bf00ffc..502b1f8faae 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GuestOSResponse; +import org.apache.cloudstack.acl.RoleType; import com.cloud.event.EventTypes; import com.cloud.storage.GuestOS; @@ -56,8 +57,10 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, required = true, description = "Map of (key/value pairs)") private Map details; + @Parameter(name="forDisplay", type=CommandType.BOOLEAN, description="whether this guest OS is available for end users", authorized = {RoleType.Admin}) + private Boolean display; -///////////////////////////////////////////////////// + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -70,7 +73,7 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { } public Map getDetails() { - Map detailsMap = new HashMap();; + Map detailsMap = new HashMap<>();; if (!details.isEmpty()) { Collection servicesCollection = details.values(); Iterator iter = servicesCollection.iterator(); @@ -84,6 +87,10 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { return detailsMap; } + public Boolean getForDisplay() { + return display; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java index 7419cd8f759..9d6cd438561 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java @@ -19,8 +19,10 @@ package org.apache.cloudstack.api.command.user.guest; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.BooleanUtils; import org.apache.log4j.Logger; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; @@ -51,6 +53,10 @@ public class ListGuestOsCmd extends BaseListCmd { @Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, description = "list os by description", since = "3.0.1") private String description; + @Parameter(name = ApiConstants.FOR_DISPLAY, type = CommandType.BOOLEAN, description = "list resources by display flag; only ROOT admin is eligible to pass this parameter", + since = "4.18.1", authorized = {RoleType.Admin}) + private Boolean display; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -67,6 +73,10 @@ public class ListGuestOsCmd extends BaseListCmd { return description; } + public Boolean getDisplay() { + return BooleanUtils.toBooleanDefaultIfNull(display, true); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -74,8 +84,8 @@ public class ListGuestOsCmd extends BaseListCmd { @Override public void execute() { Pair, Integer> result = _mgr.listGuestOSByCriteria(this); - ListResponse response = new ListResponse(); - List osResponses = new ArrayList(); + ListResponse response = new ListResponse<>(); + List osResponses = new ArrayList<>(); for (GuestOS guestOS : result.first()) { GuestOSResponse guestOSResponse = _responseGenerator.createGuestOSResponse(guestOS); osResponses.add(guestOSResponse); diff --git a/api/src/main/java/org/apache/cloudstack/api/response/GuestOSResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/GuestOSResponse.java index c1a57c3e0e1..7ec2677ca32 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/GuestOSResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/GuestOSResponse.java @@ -43,6 +43,10 @@ public class GuestOSResponse extends BaseResponse { @Param(description = "is the guest OS user defined") private Boolean isUserDefined; + @SerializedName(ApiConstants.FOR_DISPLAY) + @Param(description = "is the guest OS visible for the users") + private Boolean forDisplay; + public String getId() { return id; } @@ -75,4 +79,12 @@ public class GuestOSResponse extends BaseResponse { this.isUserDefined = isUserDefined; } + public Boolean getForDisplay() { + return this.forDisplay; + } + + public void setForDisplay(final Boolean forDisplay) { + this.forDisplay = forDisplay; + } + } diff --git a/engine/schema/src/main/java/com/cloud/storage/GuestOSVO.java b/engine/schema/src/main/java/com/cloud/storage/GuestOSVO.java index f04f9a4208c..03955f17667 100644 --- a/engine/schema/src/main/java/com/cloud/storage/GuestOSVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/GuestOSVO.java @@ -57,6 +57,9 @@ public class GuestOSVO implements GuestOS { @Column(name = "is_user_defined") private boolean isUserDefined; + @Column(name = "display", updatable = true, nullable = false) + protected boolean display = true; + @Override public long getId() { return id; @@ -120,4 +123,15 @@ public class GuestOSVO implements GuestOS { public void setIsUserDefined(boolean isUserDefined) { this.isUserDefined = isUserDefined; } + + public boolean getForDisplay() { + return isDisplay(); + } + public boolean isDisplay() { + return display; + } + + public void setDisplay(boolean display) { + this.display = display; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java index 83e19b17e25..eeae0bdae00 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java @@ -16,12 +16,23 @@ // under the License. package com.cloud.storage.dao; +import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSVO; +import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; +import java.util.List; +import java.util.Set; + public interface GuestOSDao extends GenericDao { - GuestOSVO listByDisplayName(String displayName); + GuestOSVO findOneByDisplayName(String displayName); GuestOSVO findByCategoryIdAndDisplayNameOrderByCreatedDesc(long categoryId, String displayName); + + Set findDoubleNames(); + + List listByDisplayName(String displayName); + + Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java index 68da2b92acb..19c4e902188 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java @@ -17,8 +17,19 @@ package com.cloud.storage.dao; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import com.cloud.storage.GuestOS; +import com.cloud.utils.Pair; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -42,7 +53,7 @@ public class GuestOSDaoImpl extends GenericDaoBase implements G } @Override - public GuestOSVO listByDisplayName(String displayName) { + public GuestOSVO findOneByDisplayName(String displayName) { SearchCriteria sc = Search.create(); sc.setParameters("display_name", displayName); return findOneBy(sc); @@ -62,4 +73,69 @@ public class GuestOSDaoImpl extends GenericDaoBase implements G } return null; } + + /** + + "select display_name from" + + "(select display_name, count(1) as count from guest_os go1 where removed is null group by display_name having count > 1) tab0"; + * + * @return + */ + @Override + @DB + public Set findDoubleNames() { + String selectSql = "SELECT display_name FROM (SELECT display_name, count(1) AS count FROM guest_os go1 WHERE removed IS NULL GROUP BY display_name HAVING count > 1) tab0"; + Set names = new HashSet<>(); + Connection conn = TransactionLegacy.getStandaloneConnection(); + try { + PreparedStatement stmt = conn.prepareStatement(selectSql); + ResultSet rs = stmt.executeQuery(); + while (rs != null && rs.next()) { + names.add(rs.getString(1)); + } + } catch (SQLException ex) { + throw new CloudRuntimeException("Error while trying to find duplicate guest OSses", ex); + } + return names; + } + + /** + * get all with a certain display name + * @param displayName + * @return a list with GuestOS objects + */ + @Override + public List listByDisplayName(String displayName) { + SearchCriteria sc = Search.create(); + sc.setParameters("display_name", displayName); + return listBy(sc); + } + + public Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay) { + final Filter searchFilter = new Filter(GuestOSVO.class, "displayName", true, startIndex, pageSize); + final SearchCriteria sc = createSearchCriteria(); + + if (id != null) { + sc.addAnd("id", SearchCriteria.Op.EQ, id); + } + + if (osCategoryId != null) { + sc.addAnd("categoryId", SearchCriteria.Op.EQ, osCategoryId); + } + + if (description != null) { + sc.addAnd("displayName", SearchCriteria.Op.LIKE, "%" + description + "%"); + } + + if (keyword != null) { + sc.addAnd("displayName", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + } + + if (forDisplay != null) { + sc.addAnd("display", SearchCriteria.Op.EQ, forDisplay); + } + + final Pair, Integer> result = searchAndCount(sc, searchFilter); + return new Pair<>(result.first(), result.second()); + } + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java index 47a71433a32..f9a91c7d210 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDao.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.storage.dao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.utils.db.GenericDao; @@ -24,7 +23,12 @@ import java.util.List; public interface GuestOSHypervisorDao extends GenericDao { - HypervisorType findHypervisorTypeByGuestOsId(long guestOsId); + /** + * list all the mappings for a guesos id + * @param guestOsId the guestos to look for + * @return a list of mappings + */ + List listByGuestOsId(long guestOsId); GuestOSHypervisorVO findByOsIdAndHypervisor(long guestOsId, String hypervisorType, String hypervisorVersion); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java index 65f17c29fbc..e03e3f7ce64 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSHypervisorDaoImpl.java @@ -20,13 +20,12 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; -import com.cloud.utils.db.QueryBuilder; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; -import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.GuestOSHypervisorVO; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -82,11 +81,10 @@ public class GuestOSHypervisorDaoImpl extends GenericDaoBase listByGuestOsId(long guestOsId) { SearchCriteria sc = guestOsSearch.create(); sc.setParameters("guest_os_id", guestOsId); - GuestOSHypervisorVO goh = findOneBy(sc); - return HypervisorType.getType(goh.getHypervisorType()); + return listBy(sc); } @Override diff --git a/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java b/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java index 739eb32261c..20f9d851c3b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/GuestOsMapper.java @@ -23,7 +23,9 @@ import org.apache.log4j.Logger; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.inject.Inject; @@ -53,6 +55,89 @@ public class GuestOsMapper { guestOSDao = new GuestOSDaoImpl(); } + public void mergeDuplicates() { + LOG.info("merging duplicate guest osses"); + Set> duplicates = findDuplicates(); + LOG.debug(String.format("merging %d sets of duplicates", duplicates.size())); + for (Set setOfGuestOSes : duplicates) { + // decide which to (mark as) remove(d) + // # highest/lowest id + // # or is user_defined == false + GuestOSVO guestOSVO = highestIdFrom(setOfGuestOSes); + LOG.info(String.format("merging %d duplicates for %s ", setOfGuestOSes.size(), guestOSVO.getDisplayName())); + makeNormative(guestOSVO, setOfGuestOSes); + + } + } + + public void makeNormative(GuestOSVO guestOSVO, Set setOfGuestOSes) { + for (GuestOSVO oldGuestOs : setOfGuestOSes) { + if (guestOSVO.getId() != oldGuestOs.getId()) { + List mappings = guestOSHypervisorDao.listByGuestOsId(oldGuestOs.getId()); + copyMappings(guestOSVO, mappings); + makeHidden(oldGuestOs); + } + } + } + + private void makeHidden(GuestOSVO guestOSVO) { + guestOSVO.setDisplay(false); + guestOSDao.update(guestOSVO.getId(),guestOSVO); + } + + private void copyMappings(GuestOSVO guestOSVO, List mappings) { + for (GuestOSHypervisorVO mapping : mappings) { + if (null == guestOSHypervisorDao.findByOsIdAndHypervisor(guestOSVO.getId(), mapping.getHypervisorType(), mapping.getHypervisorVersion())) { + GuestOSHypervisorVO newMap = new GuestOSHypervisorVO(); + newMap.setGuestOsId(guestOSVO.getId()); + newMap.setGuestOsName(mapping.getGuestOsName()); + newMap.setHypervisorType(mapping.getHypervisorType()); + newMap.setHypervisorVersion(mapping.getHypervisorVersion()); + guestOSHypervisorDao.persist(newMap); + } + } + } + + private GuestOSVO highestIdFrom(Set setOfGuestOSes) { + GuestOSVO rc = null; + for (GuestOSVO guestOSVO: setOfGuestOSes) { + if (rc == null || (guestOSVO.getId() > rc.getId() && !guestOSVO.getIsUserDefined())) { + rc = guestOSVO; + break; + } + } + return rc; + } + + /** + * + ¨¨¨ + select * from guest_os go2 + where display_name + in (select display_name from + (select display_name, count(1) as count from guest_os go1 group by display_name having count > 1) tab0); + ¨¨¨ + * and group them by display_name + * + * + * @return a list of sets of duplicate + */ + private Set> findDuplicates() { + Set> rc = new HashSet<>(); + Set names = guestOSDao.findDoubleNames(); + for (String name : names) { + List guestOsses = guestOSDao.listByDisplayName(name); + if (CollectionUtils.isNotEmpty(guestOsses)) { + rc.add(new HashSet<>(guestOsses)); + } + } + return rc; + } + + public List listByDisplayName(String displayName) { + return guestOSDao.listByDisplayName(displayName); + } + private long getGuestOsId(long categoryId, String displayName) { GuestOSVO guestOS = guestOSDao.findByCategoryIdAndDisplayNameOrderByCreatedDesc(categoryId, displayName); long id = 0l; @@ -76,26 +161,33 @@ public class GuestOsMapper { } public void addGuestOsAndHypervisorMappings(long categoryId, String displayName, List mappings) { - if (!addGuestOs(categoryId, displayName)) { - LOG.warn("Couldn't add the guest OS with category id: " + categoryId + " and display name: " + displayName); - return; + long guestOsId = getGuestOsId(categoryId, displayName); + if (guestOsId == 0) { + LOG.debug("No guest OS found with category id: " + categoryId + " and display name: " + displayName); + if (!addGuestOs(categoryId, displayName)) { + LOG.warn("Couldn't add the guest OS with category id: " + categoryId + " and display name: " + displayName); + return; + } + guestOsId = getGuestOsId(categoryId, displayName); + } else { + updateToSystemDefined(guestOsId); } if (CollectionUtils.isEmpty(mappings)) { return; } - long guestOsId = getGuestOsId(categoryId, displayName); - if (guestOsId == 0) { - LOG.debug("No guest OS found with category id: " + categoryId + " and display name: " + displayName); - return; - } - for (final GuestOSHypervisorMapping mapping : mappings) { addGuestOsHypervisorMapping(mapping, guestOsId); } } + private void updateToSystemDefined(long guestOsId) { + GuestOSVO guestOsVo = guestOSDao.findById(guestOsId); + guestOsVo.setIsUserDefined(false); + guestOSDao.update(guestOsId, guestOsVo);// TODO: update is_user_defined to false + } + public boolean addGuestOs(long categoryId, String displayName) { LOG.debug("Adding guest OS with category id: " + categoryId + " and display name: " + displayName); GuestOSVO guestOS = new GuestOSVO(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java index 53bd497ca22..8eb50666ec8 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java @@ -19,6 +19,7 @@ package com.cloud.upgrade.dao; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.GuestOSHypervisorMapping; import com.cloud.upgrade.GuestOsMapper; +import com.cloud.storage.GuestOSVO; import com.cloud.upgrade.SystemVmTemplateRegistration; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.log4j.Logger; @@ -26,10 +27,13 @@ import org.apache.log4j.Logger; import java.io.InputStream; import java.sql.Connection; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate { final static Logger LOG = Logger.getLogger(Upgrade41800to41810.class); + private GuestOsMapper guestOsMapper = new GuestOsMapper(); + private SystemVmTemplateRegistration systemVmTemplateRegistration; @Override @@ -64,6 +68,14 @@ public class Upgrade41800to41810 implements DbUpgrade, DbUpgradeSystemVmTemplate updateGuestOsMappings(conn); copyGuestOsMappingsToVMware80u1(); addForeignKeyToAutoscaleVmprofiles(conn); + mergeDuplicateGuestOSes(); + } + + private void mergeDuplicateGuestOSes() { + guestOsMapper.mergeDuplicates(); + List nines = guestOsMapper.listByDisplayName("Red Hat Enterprise Linux 9"); + GuestOSVO nineDotZero = guestOsMapper.listByDisplayName("Red Hat Enterprise Linux 9.0").get(0); + guestOsMapper.makeNormative(nineDotZero, new HashSet<>(nines)); } @Override diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql index 2af6723c134..7941424a43f 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql @@ -1568,4 +1568,6 @@ UPDATE SET usage_type = 22 WHERE - usage_type = 24 AND usage_display like '% io write'; \ No newline at end of file + usage_type = 24 AND usage_display like '% io write'; + +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os', 'display', 'tinyint(1) DEFAULT ''1'' COMMENT ''should this guest_os be shown to the end user'' '); \ No newline at end of file diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql index 495c184918c..4ea2bf57292 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41800to41810.sql @@ -43,3 +43,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.autoscale_vmprofiles', 'user_data_de UPDATE `cloud`.`service_offering` so SET so.limit_cpu_use = 0 WHERE so.default_use = 1 AND so.vm_type IN ('domainrouter', 'secondarystoragevm', 'consoleproxy', 'internalloadbalancervm', 'elasticloadbalancervm'); + +-- fix erronous commas in guest_os names +UPDATE `cloud`.`guest_os_hypervisor` SET guest_os_name = 'rhel9_64Guest' WHERE guest_os_name = 'rhel9_64Guest,'; +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os', 'display', 'tinyint(1) DEFAULT ''1'' COMMENT ''should this guest_os be shown to the end user'' '); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java index f148a16473e..db41ab19d56 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/guru/VMwareGuru.java @@ -456,8 +456,8 @@ public class VMwareGuru extends HypervisorGuruBase implements HypervisorGuru, Co */ private Long getImportingVMGuestOs(VirtualMachineConfigSummary configSummary) { String guestFullName = configSummary.getGuestFullName(); - GuestOSVO os = _guestOsDao.listByDisplayName(guestFullName); - return os != null ? os.getId() : _guestOsDao.listByDisplayName("Other (64-bit)").getId(); + GuestOSVO os = _guestOsDao.findOneByDisplayName(guestFullName); + return os != null ? os.getId() : _guestOsDao.findOneByDisplayName("Other (64-bit)").getId(); } /** diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 162b1f38add..c4d7f3cc717 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -1104,7 +1104,7 @@ public class ApiDBUtils { } public static GuestOS findGuestOSByDisplayName(String displayName) { - return s_guestOSDao.listByDisplayName(displayName); + return s_guestOSDao.findOneByDisplayName(displayName); } public static HostVO findHostById(Long hostId) { diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 55f00a609e9..05aee6b6ccb 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -3646,6 +3646,7 @@ public class ApiResponseHelper implements ResponseGenerator { response.setDescription(guestOS.getDisplayName()); response.setId(guestOS.getUuid()); response.setIsUserDefined(guestOS.getIsUserDefined()); + response.setForDisplay(guestOS.getForDisplay()); GuestOSCategoryVO category = ApiDBUtils.findGuestOsCategoryById(guestOS.getCategoryId()); if (category != null) { response.setOsCategoryId(category.getUuid()); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 85e86b5b21b..c6ace852bd3 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -2625,32 +2625,15 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Override public Pair, Integer> listGuestOSByCriteria(final ListGuestOsCmd cmd) { - final Filter searchFilter = new Filter(GuestOSVO.class, "displayName", true, cmd.getStartIndex(), cmd.getPageSizeVal()); final Long id = cmd.getId(); final Long osCategoryId = cmd.getOsCategoryId(); final String description = cmd.getDescription(); final String keyword = cmd.getKeyword(); + final Long startIndex = cmd.getStartIndex(); + final Long pageSize = cmd.getPageSizeVal(); + Boolean forDisplay = cmd.getDisplay(); - final SearchCriteria sc = _guestOSDao.createSearchCriteria(); - - if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); - } - - if (osCategoryId != null) { - sc.addAnd("categoryId", SearchCriteria.Op.EQ, osCategoryId); - } - - if (description != null) { - sc.addAnd("displayName", SearchCriteria.Op.LIKE, "%" + description + "%"); - } - - if (keyword != null) { - sc.addAnd("displayName", SearchCriteria.Op.LIKE, "%" + keyword + "%"); - } - - final Pair, Integer> result = _guestOSDao.searchAndCount(sc, searchFilter); - return new Pair, Integer>(result.first(), result.second()); + return _guestOSDao.listGuestOSByCriteria(startIndex, pageSize, id, osCategoryId, description, keyword, forDisplay); } @Override @@ -2796,18 +2779,20 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe guestOsVo.setDisplayName(displayName); guestOsVo.setName(name); guestOsVo.setIsUserDefined(true); + guestOsVo.setDisplay(cmd.getForDisplay() == null ? true : cmd.getForDisplay()); final GuestOS guestOsPersisted = _guestOSDao.persist(guestOsVo); - if (cmd.getDetails() != null && !cmd.getDetails().isEmpty()) { - Map detailsMap = cmd.getDetails(); - for (Object key : detailsMap.keySet()) { - _guestOsDetailsDao.addDetail(guestOsPersisted.getId(), (String)key, detailsMap.get(key), false); - } - } + persistGuestOsDetails(cmd.getDetails(), guestOsPersisted.getId()); return guestOsPersisted; } + private void persistGuestOsDetails(Map details, long guestOsPersistedId) { + for (Object key : details.keySet()) { + _guestOsDetailsDao.addDetail(guestOsPersistedId, (String)key, details.get(key), false); + } + } + @Override @ActionEvent(eventType = EventTypes.EVENT_GUEST_OS_ADD, eventDescription = "Adding a new guest OS type", async = true) public GuestOS getAddedGuestOs(final Long guestOsId) { @@ -2831,12 +2816,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe throw new InvalidParameterValueException("Unable to modify system defined guest OS"); } - if (cmd.getDetails() != null && !cmd.getDetails().isEmpty()) { - Map detailsMap = cmd.getDetails(); - for (Object key : detailsMap.keySet()) { - _guestOsDetailsDao.addDetail(id, (String)key, detailsMap.get(key), false); - } - } + persistGuestOsDetails(cmd.getDetails(), id); //Check if update is needed if (displayName.equals(guestOsHandle.getDisplayName())) { @@ -2850,6 +2830,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } final GuestOSVO guestOs = _guestOSDao.createForUpdate(id); guestOs.setDisplayName(displayName); + if (cmd.getForDisplay() != null) { + guestOs.setDisplay(cmd.getForDisplay()); + } if (_guestOSDao.update(id, guestOs)) { return _guestOSDao.findById(id); } else { diff --git a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java index 72ccf621e3a..eab1e98c800 100644 --- a/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManagerImpl.java @@ -1214,7 +1214,7 @@ public class UnmanagedVMsManagerImpl implements UnmanagedVMsManager { String osName = unmanagedInstance.getOperatingSystem(); GuestOS guestOS = null; if (StringUtils.isNotEmpty(osName)) { - guestOS = guestOSDao.listByDisplayName(osName); + guestOS = guestOSDao.findOneByDisplayName(osName); } GuestOSHypervisor guestOSHypervisor = null; if (guestOS != null) { From 8cb30551d584462c697a4f49c9106d4f15b5cd53 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 23 Aug 2023 12:36:16 +0200 Subject: [PATCH 29/31] UI: update ip ranges of shared networks (#7896) * UI: update ip ranges of shared networks * UI: remove ip6gateway and ip6cidr from GuestIpRanges.vue --- ui/src/views/network/CreateVlanIpRange.vue | 4 +- ui/src/views/network/GuestIpRanges.vue | 98 ++++++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/ui/src/views/network/CreateVlanIpRange.vue b/ui/src/views/network/CreateVlanIpRange.vue index 96bbc9c041b..d69c1451fe8 100644 --- a/ui/src/views/network/CreateVlanIpRange.vue +++ b/ui/src/views/network/CreateVlanIpRange.vue @@ -195,14 +195,14 @@ export default { { validator: this.checkIpFormat, ipV6: true, - message: this.$t('message.error.ipV6.address') + message: this.$t('message.error.ipv6.address') } ], endipv6: [ { validator: this.checkIpFormat, ipV6: true, - message: this.$t('message.error.ipV6.address') + message: this.$t('message.error.ipv6.address') } ] }) diff --git a/ui/src/views/network/GuestIpRanges.vue b/ui/src/views/network/GuestIpRanges.vue index 14cd8a116bc..b2e56de5820 100644 --- a/ui/src/views/network/GuestIpRanges.vue +++ b/ui/src/views/network/GuestIpRanges.vue @@ -37,6 +37,12 @@ :pagination="false" >