From b4325eccfb44ea1d842d1fe9e836cae0f31e2b1a Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 29 Aug 2024 06:09:14 -0300 Subject: [PATCH 01/45] Fix userdata append header restrictions (#9575) --- .../userdata/CloudInitUserDataProvider.java | 28 ++++++++----- .../CloudInitUserDataProviderTest.java | 39 +++++++++++++++++-- ui/src/utils/plugins.js | 2 +- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java b/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java index 65996f181a9..feca8712a7d 100644 --- a/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java +++ b/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java @@ -88,14 +88,20 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr .filter(x -> (x.startsWith("#") && !x.startsWith("##")) || (x.startsWith("Content-Type:"))) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(lines)) { - throw new CloudRuntimeException("Failed to detect the user data format type as it " + - "does not contain a header"); + LOGGER.debug("Failed to detect the user data format type as it does not contain a header"); + return null; } return lines.get(0); } - protected FormatType mapUserDataHeaderToFormatType(String header) { - if (header.equalsIgnoreCase("#cloud-config")) { + protected FormatType mapUserDataHeaderToFormatType(String header, FormatType defaultFormatType) { + if (StringUtils.isBlank(header)) { + if (defaultFormatType == null) { + throw new CloudRuntimeException("Failed to detect the user data format type as it does not contain a header"); + } + LOGGER.debug(String.format("Empty header for userdata, using the default format type: %s", defaultFormatType.name())); + return defaultFormatType; + } else if (header.equalsIgnoreCase("#cloud-config")) { return FormatType.CLOUD_CONFIG; } else if (header.startsWith("#!")) { return FormatType.BASH_SCRIPT; @@ -115,9 +121,11 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr /** * Detect the user data type + * @param userdata the userdata string to detect the type + * @param defaultFormatType if not null, then use it in case the header does not exist in the userdata, otherwise fail * Reference: */ - protected FormatType getUserDataFormatType(String userdata) { + protected FormatType getUserDataFormatType(String userdata, FormatType defaultFormatType) { if (StringUtils.isBlank(userdata)) { String msg = "User data expected but provided empty user data"; LOGGER.error(msg); @@ -125,7 +133,7 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr } String header = extractUserDataHeader(userdata); - return mapUserDataHeaderToFormatType(header); + return mapUserDataHeaderToFormatType(header, defaultFormatType); } private String getContentType(String userData, FormatType formatType) throws MessagingException { @@ -234,7 +242,9 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr } private String simpleAppendSameFormatTypeUserData(String userData1, String userData2) { - return String.format("%s\n\n%s", userData1, userData2.substring(userData2.indexOf('\n')+1)); + String userdata2Header = extractUserDataHeader(userData2); + int beginIndex = StringUtils.isNotBlank(userdata2Header) ? userData2.indexOf('\n')+1 : 0; + return String.format("%s\n\n%s", userData1, userData2.substring(beginIndex)); } private void checkGzipAppend(String encodedUserData1, String encodedUserData2) { @@ -249,8 +259,8 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr checkGzipAppend(encodedUserData1, encodedUserData2); String userData1 = new String(Base64.decodeBase64(encodedUserData1)); String userData2 = new String(Base64.decodeBase64(encodedUserData2)); - FormatType formatType1 = getUserDataFormatType(userData1); - FormatType formatType2 = getUserDataFormatType(userData2); + FormatType formatType1 = getUserDataFormatType(userData1, null); + FormatType formatType2 = getUserDataFormatType(userData2, formatType1); if (formatType1.equals(formatType2) && List.of(FormatType.CLOUD_CONFIG, FormatType.BASH_SCRIPT).contains(formatType1)) { return simpleAppendSameFormatTypeUserData(userData1, userData2); } diff --git a/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java b/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java index 4ca9fb7ebd6..86b6a6fb6ea 100644 --- a/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java +++ b/engine/userdata/cloud-init/src/test/java/org/apache/cloudstack/userdata/CloudInitUserDataProviderTest.java @@ -73,21 +73,28 @@ public class CloudInitUserDataProviderTest { @Test public void testGetUserDataFormatType() { - CloudInitUserDataProvider.FormatType type = provider.getUserDataFormatType(CLOUD_CONFIG_USERDATA); + CloudInitUserDataProvider.FormatType type = provider.getUserDataFormatType(CLOUD_CONFIG_USERDATA, null); Assert.assertEquals(CloudInitUserDataProvider.FormatType.CLOUD_CONFIG, type); } @Test(expected = CloudRuntimeException.class) public void testGetUserDataFormatTypeNoHeader() { String userdata = "password: password\nchpasswd: { expire: False }\nssh_pwauth: True"; - provider.getUserDataFormatType(userdata); + provider.getUserDataFormatType(userdata, null); + } + + @Test + public void testGetUserDataFormatTypeNoHeaderDefaultFormat() { + String userdata = "password: password\nchpasswd: { expire: False }\nssh_pwauth: True"; + CloudInitUserDataProvider.FormatType defaultFormatType = CloudInitUserDataProvider.FormatType.CLOUD_CONFIG; + Assert.assertEquals(defaultFormatType, provider.getUserDataFormatType(userdata, defaultFormatType)); } @Test(expected = CloudRuntimeException.class) public void testGetUserDataFormatTypeInvalidType() { String userdata = "#invalid-type\n" + "password: password\nchpasswd: { expire: False }\nssh_pwauth: True"; - provider.getUserDataFormatType(userdata); + provider.getUserDataFormatType(userdata, null); } private MimeMultipart getCheckedMultipartFromMultipartData(String multipartUserData, int count) { @@ -111,6 +118,16 @@ public class CloudInitUserDataProviderTest { getCheckedMultipartFromMultipartData(multipartUserData, 2); } + @Test + public void testAppendUserDataSecondWithoutHeader() { + String userdataWithHeader = Base64.encodeBase64String(SHELL_SCRIPT_USERDATA1.getBytes()); + String bashScriptWithoutHeader = "echo \"without header\""; + String userdataWithoutHeader = Base64.encodeBase64String(bashScriptWithoutHeader.getBytes()); + String appended = provider.appendUserData(userdataWithHeader, userdataWithoutHeader); + String expected = String.format("%s\n\n%s", SHELL_SCRIPT_USERDATA1, bashScriptWithoutHeader); + Assert.assertEquals(expected, appended); + } + @Test public void testAppendSameShellScriptTypeUserData() { String result = SHELL_SCRIPT_USERDATA + "\n\n" + @@ -129,6 +146,22 @@ public class CloudInitUserDataProviderTest { Assert.assertEquals(result, appendUserData); } + @Test + public void testAppendCloudConfig() { + String userdata1 = "#cloud-config\n" + + "chpasswd:\n" + + " list: |\n" + + " root:password\n" + + " expire: False"; + String userdata2 = "write_files:\n" + + "- path: /root/CLOUD_INIT_WAS_HERE"; + String userdataWithHeader = Base64.encodeBase64String(userdata1.getBytes()); + String userdataWithoutHeader = Base64.encodeBase64String(userdata2.getBytes()); + String appended = provider.appendUserData(userdataWithHeader, userdataWithoutHeader); + String expected = String.format("%s\n\n%s", userdata1, userdata2); + Assert.assertEquals(expected, appended); + } + @Test public void testAppendUserDataMIMETemplateData() { String multipartUserData = provider.appendUserData( diff --git a/ui/src/utils/plugins.js b/ui/src/utils/plugins.js index e358988fac9..6beda22a8da 100644 --- a/ui/src/utils/plugins.js +++ b/ui/src/utils/plugins.js @@ -506,7 +506,7 @@ export const genericUtilPlugin = { if (isBase64(text)) { return text } - return encodeURIComponent(btoa(unescape(encodeURIComponent(text)))) + return encodeURI(btoa(unescape(encodeURIComponent(text)))) } } } From 0204cb75e371770085c09dc48d95ec2c10010d61 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Aug 2024 18:45:33 +0530 Subject: [PATCH 02/45] ui: show guest networks for guest vlans list (#9554) Signed-off-by: Abhishek Kumar --- ui/src/components/view/ListView.vue | 6 ++++++ ui/src/config/section/network.js | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index ff2e385f03c..087c547aa69 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -240,6 +240,12 @@ + diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index 8399d9769ef..fd1b15557af 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -1364,7 +1364,7 @@ export default { permission: ['listGuestVlans'], resourceType: 'GuestVlan', filters: ['allocatedonly', 'all'], - columns: ['vlan', 'allocationstate', 'physicalnetworkname', 'taken', 'account', 'project', 'domain', 'zonename'], + columns: ['vlan', 'allocationstate', 'physicalnetworkname', 'taken', 'account', 'project', 'domain', 'zonename', 'guest.networks'], details: ['vlan', 'allocationstate', 'physicalnetworkname', 'taken', 'account', 'project', 'domain', 'isdedicated', 'zonename'], searchFilters: ['zoneid'], tabs: [{ From a5f55602fc3c21a56715c2b944f31c2b67f134ed Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Fri, 30 Aug 2024 08:51:12 +0200 Subject: [PATCH 03/45] LOGGER -> logger in CloudInitUserDataProvider.java --- .../apache/cloudstack/userdata/CloudInitUserDataProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java b/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java index 6f633f6abb2..02e6adcc784 100644 --- a/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java +++ b/engine/userdata/cloud-init/src/main/java/org/apache/cloudstack/userdata/CloudInitUserDataProvider.java @@ -85,7 +85,7 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr .filter(x -> (x.startsWith("#") && !x.startsWith("##")) || (x.startsWith("Content-Type:"))) .collect(Collectors.toList()); if (CollectionUtils.isEmpty(lines)) { - LOGGER.debug("Failed to detect the user data format type as it does not contain a header"); + logger.debug("Failed to detect the user data format type as it does not contain a header"); return null; } return lines.get(0); @@ -96,7 +96,7 @@ public class CloudInitUserDataProvider extends AdapterBase implements UserDataPr if (defaultFormatType == null) { throw new CloudRuntimeException("Failed to detect the user data format type as it does not contain a header"); } - LOGGER.debug(String.format("Empty header for userdata, using the default format type: %s", defaultFormatType.name())); + logger.debug(String.format("Empty header for userdata, using the default format type: %s", defaultFormatType.name())); return defaultFormatType; } else if (header.equalsIgnoreCase("#cloud-config")) { return FormatType.CLOUD_CONFIG; From 8c301b16ff376f7a28684962524c6941877e2fed Mon Sep 17 00:00:00 2001 From: Felipe <124818914+FelipeM525@users.noreply.github.com> Date: Mon, 2 Sep 2024 05:17:42 -0300 Subject: [PATCH 04/45] fixed incorrect label in vrs and svms (#9617) --- ui/src/components/view/InfoCard.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/src/components/view/InfoCard.vue b/ui/src/components/view/InfoCard.vue index d4c0ffc77b6..24b7cce0feb 100644 --- a/ui/src/components/view/InfoCard.vue +++ b/ui/src/components/view/InfoCard.vue @@ -532,7 +532,8 @@
-
{{ $t('label.serviceofferingname') }}
+
{{ $t('label.system.offering') }}
+
{{ $t('label.serviceofferingname') }}
{{ resource.serviceofferingname || resource.serviceofferingid }} From abaf4b52ad18b71aea53604278f302a403413d30 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Mon, 2 Sep 2024 21:04:06 -0300 Subject: [PATCH 05/45] Fix VGPU available devices listing (#9573) * Fix VGPU available devices listing * Missing space * Refactor --- .../src/main/java/com/cloud/utils/db/Filter.java | 14 +++++++++++++- .../com/cloud/resource/ResourceManagerImpl.java | 3 ++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/framework/db/src/main/java/com/cloud/utils/db/Filter.java b/framework/db/src/main/java/com/cloud/utils/db/Filter.java index 15161ab058f..fb8c9ee37fc 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/Filter.java +++ b/framework/db/src/main/java/com/cloud/utils/db/Filter.java @@ -22,6 +22,7 @@ import javax.persistence.Column; import com.cloud.utils.Pair; import com.cloud.utils.ReflectUtil; +import org.apache.commons.lang3.StringUtils; /** * Try to use static initialization to help you in finding incorrect @@ -59,6 +60,11 @@ public class Filter { _orderBy = " ORDER BY RAND() LIMIT " + limit; } + public Filter(Long offset, Long limit) { + _offset = offset; + _limit = limit; + } + /** * Note that this copy constructor does not copy offset and limit. * @param that filter @@ -70,6 +76,10 @@ public class Filter { } public void addOrderBy(Class clazz, String field, boolean ascending) { + addOrderBy(clazz, field, ascending, null); + } + + public void addOrderBy(Class clazz, String field, boolean ascending, String tableAlias) { if (field == null) { return; } @@ -83,7 +93,9 @@ public class Filter { String name = column != null ? column.name() : field; StringBuilder order = new StringBuilder(); - if (column == null || column.table() == null || column.table().length() == 0) { + if (StringUtils.isNotBlank(tableAlias)) { + order.append(tableAlias); + } else if (column == null || column.table() == null || column.table().length() == 0) { order.append(DbUtil.getTableName(clazz)); } else { order.append(column.table()); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 5d3ec62c56d..5ba79c2d7bd 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -3417,7 +3417,8 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public List listAvailableGPUDevice(final long hostId, final String groupName, final String vgpuType) { - final Filter searchFilter = new Filter(VGPUTypesVO.class, "remainingCapacity", false, null, null); + Filter searchFilter = new Filter(null, null); + searchFilter.addOrderBy(VGPUTypesVO.class, "remainingCapacity", false, "groupId"); final SearchCriteria sc = _gpuAvailability.create(); sc.setParameters("hostId", hostId); sc.setParameters("groupName", groupName); From 929cfbc3e26d6865163f651c05f03742c36601b8 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Tue, 3 Sep 2024 12:58:16 +0200 Subject: [PATCH 06/45] Update to Debian 12 (#9627) --- .../systemvmtemplate/scripts/configure_systemvm_services.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh index 1a465f4999f..1df64d114cb 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh @@ -41,7 +41,7 @@ function configure_issue() { __?.o/ Apache CloudStack SystemVM $CLOUDSTACK_RELEASE ( )# https://cloudstack.apache.org - (___(_) Debian GNU/Linux 11 \n \l + (___(_) Debian GNU/Linux 12 \n \l EOF } From 537c0a1e8d2cbe12f3e510d6b51babd6486f4c3e Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Tue, 3 Sep 2024 13:01:07 +0200 Subject: [PATCH 07/45] linstor: set/unset allow-two-primaries and protocol on rc level (#9560) --- plugins/storage/volume/linstor/CHANGELOG.md | 12 ++++ .../kvm/storage/LinstorStorageAdaptor.java | 58 +++++++++++++------ .../storage/datastore/util/LinstorUtil.java | 13 +++-- 3 files changed, 60 insertions(+), 23 deletions(-) create mode 100644 plugins/storage/volume/linstor/CHANGELOG.md diff --git a/plugins/storage/volume/linstor/CHANGELOG.md b/plugins/storage/volume/linstor/CHANGELOG.md new file mode 100644 index 00000000000..30f0225b45e --- /dev/null +++ b/plugins/storage/volume/linstor/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to Linstor CloudStack plugin will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [2024-08-27] + +### Changed + +- Allow two primaries(+protocol c) is now set on resource-connection level instead of rd diff --git a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java index b17c768b96a..0ae65573d2b 100644 --- a/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java +++ b/plugins/storage/volume/linstor/src/main/java/com/cloud/hypervisor/kvm/storage/LinstorStorageAdaptor.java @@ -19,6 +19,8 @@ package com.cloud.hypervisor.kvm.storage; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; + +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -37,6 +39,7 @@ import org.libvirt.LibvirtException; import com.cloud.storage.Storage; import com.cloud.utils.exception.CloudRuntimeException; + import com.linbit.linstor.api.ApiClient; import com.linbit.linstor.api.ApiConsts; import com.linbit.linstor.api.ApiException; @@ -47,8 +50,8 @@ import com.linbit.linstor.api.model.ApiCallRcList; import com.linbit.linstor.api.model.Properties; import com.linbit.linstor.api.model.ProviderKind; import com.linbit.linstor.api.model.Resource; +import com.linbit.linstor.api.model.ResourceConnectionModify; import com.linbit.linstor.api.model.ResourceDefinition; -import com.linbit.linstor.api.model.ResourceDefinitionModify; import com.linbit.linstor.api.model.ResourceGroup; import com.linbit.linstor.api.model.ResourceGroupSpawn; import com.linbit.linstor.api.model.ResourceMakeAvailable; @@ -262,15 +265,19 @@ public class LinstorStorageAdaptor implements StorageAdaptor { * @throws ApiException if any problem connecting to the Linstor controller */ private void allow2PrimariesIfInUse(DevelopersApi api, String rscName) throws ApiException { - if (LinstorUtil.isResourceInUse(api, rscName)) { + String inUseNode = LinstorUtil.isResourceInUse(api, rscName); + if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { // allow 2 primaries for live migration, should be removed by disconnect on the other end - ResourceDefinitionModify rdm = new ResourceDefinitionModify(); + ResourceConnectionModify rcm = new ResourceConnectionModify(); Properties props = new Properties(); props.put("DrbdOptions/Net/allow-two-primaries", "yes"); - rdm.setOverrideProps(props); - ApiCallRcList answers = api.resourceDefinitionModify(rscName, rdm); + props.put("DrbdOptions/Net/protocol", "C"); + rcm.setOverrideProps(props); + ApiCallRcList answers = api.resourceConnectionModify(rscName, inUseNode, localNodeName, rcm); if (answers.hasError()) { - s_logger.error("Unable to set 'allow-two-primaries' on " + rscName); + s_logger.error(String.format( + "Unable to set protocol C and 'allow-two-primaries' on %s/%s/%s", + inUseNode, localNodeName, rscName)); // do not fail here as adding allow-two-primaries property is only a problem while live migrating } } @@ -310,6 +317,23 @@ public class LinstorStorageAdaptor implements StorageAdaptor { return true; } + private void removeTwoPrimariesRcProps(DevelopersApi api, String inUseNode, String rscName) throws ApiException { + ResourceConnectionModify rcm = new ResourceConnectionModify(); + List deleteProps = new ArrayList<>(); + deleteProps.add("DrbdOptions/Net/allow-two-primaries"); + deleteProps.add("DrbdOptions/Net/protocol"); + rcm.deleteProps(deleteProps); + ApiCallRcList answers = api.resourceConnectionModify(rscName, localNodeName, inUseNode, rcm); + if (answers.hasError()) { + s_logger.error( + String.format("Failed to remove 'protocol' and 'allow-two-primaries' on %s/%s/%s: %s", + localNodeName, + inUseNode, + rscName, LinstorUtil.getBestErrorMessage(answers))); + // do not fail here as removing allow-two-primaries property isn't fatal + } + } + private boolean tryDisconnectLinstor(String volumePath, KVMStoragePool pool) { if (volumePath == null) { @@ -339,9 +363,18 @@ public class LinstorStorageAdaptor implements StorageAdaptor { if (optRsc.isPresent()) { + Resource rsc = optRsc.get(); try { - Resource rsc = optRsc.get(); + String inUseNode = LinstorUtil.isResourceInUse(api, rsc.getName()); + if (inUseNode != null && !inUseNode.equalsIgnoreCase(localNodeName)) { + removeTwoPrimariesRcProps(api, inUseNode, rsc.getName()); + } + } catch (ApiException apiEx) { + s_logger.error(apiEx.getBestMessage()); + // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal + } + try { // if diskless resource remove it, in the worst case it will be transformed to a tiebreaker if (rsc.getFlags() != null && rsc.getFlags().contains(ApiConsts.FLAG_DRBD_DISKLESS) && @@ -349,17 +382,6 @@ public class LinstorStorageAdaptor implements StorageAdaptor { ApiCallRcList delAnswers = api.resourceDelete(rsc.getName(), localNodeName); logLinstorAnswers(delAnswers); } - - // remove allow-two-primaries - ResourceDefinitionModify rdm = new ResourceDefinitionModify(); - rdm.deleteProps(Collections.singletonList("DrbdOptions/Net/allow-two-primaries")); - ApiCallRcList answers = api.resourceDefinitionModify(rsc.getName(), rdm); - if (answers.hasError()) { - s_logger.error( - String.format("Failed to remove 'allow-two-primaries' on %s: %s", - rsc.getName(), LinstorUtil.getBestErrorMessage(answers))); - // do not fail here as removing allow-two-primaries property isn't fatal - } } catch (ApiException apiEx) { s_logger.error(apiEx.getBestMessage()); // do not fail here as removing allow-two-primaries property or deleting diskless isn't fatal diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java index 9f4dbae7835..db2b5d1ea0b 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorUtil.java @@ -97,17 +97,20 @@ public class LinstorUtil { * * @param api developer api object to use * @param rscName resource name to check in use state. - * @return True if a resource found that is in use(primary) state, else false. + * @return NodeName where the resource is inUse, if not in use `null` * @throws ApiException forwards api errors */ - public static boolean isResourceInUse(DevelopersApi api, String rscName) throws ApiException { + public static String isResourceInUse(DevelopersApi api, String rscName) throws ApiException { List rscs = api.resourceList(rscName, null, null); if (rscs != null) { return rscs.stream() - .anyMatch(rsc -> rsc.getState() != null && Boolean.TRUE.equals(rsc.getState().isInUse())); - } + .filter(rsc -> rsc.getState() != null && Boolean.TRUE.equals(rsc.getState().isInUse())) + .map(Resource::getNodeName) + .findFirst() + .orElse(null); + } s_logger.error("isResourceInUse: null returned from resourceList"); - return false; + return null; } /** From 0692a296ce927fe5b41736b7a28bf739a52f4dd3 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 3 Sep 2024 17:33:57 +0530 Subject: [PATCH 08/45] engine-orchestration: fix issue for empty product in vm metadata (#9610) Signed-off-by: Rohit Yadav Signed-off-by: Abhishek Kumar Co-authored-by: Rohit Yadav --- .../java/com/cloud/vm/VirtualMachineManager.java | 2 +- .../java/com/cloud/vm/VirtualMachineManagerImpl.java | 2 +- .../com/cloud/vm/VirtualMachineManagerImplTest.java | 12 ++++++++++++ .../cloud/hypervisor/kvm/resource/LibvirtVMDef.java | 6 ++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 75211df5291..e8ffd86ac4f 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -96,7 +96,7 @@ public interface VirtualMachineManager extends Manager { ConfigKey VmMetadataProductName = new ConfigKey<>("Advanced", String.class, "vm.metadata.product", "", "If provided, a custom product name will be used in the instance metadata. When an empty" + - "value is set then default product name will be 'CloudStack Hypervisor'. " + + "value is set then default product name will be 'CloudStack Hypervisor'. " + "A custom product name may break cloud-init functionality with CloudStack datasource. Please " + "refer documentation", true, ConfigKey.Scope.Zone); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 3fa27e52c09..7c107ed6f54 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1108,7 +1108,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } vmTO.setMetadataManufacturer(metadataManufacturer); String metadataProduct = VmMetadataProductName.valueIn(vm.getDataCenterId()); - if (StringUtils.isBlank(metadataManufacturer)) { + if (StringUtils.isBlank(metadataProduct)) { metadataProduct = String.format("CloudStack %s Hypervisor", vm.getHypervisorType().toString()); } vmTO.setMetadataProductName(metadataProduct); diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index f2bd1095c81..906cded455e 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -1272,6 +1272,7 @@ public class VirtualMachineManagerImplTest { false, false, "Pass"); VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM); return new Pair<>(virtualMachineTO, vm); } @@ -1284,6 +1285,17 @@ public class VirtualMachineManagerImplTest { Assert.assertEquals(VirtualMachineManager.VmMetadataManufacturer.defaultValue(), to.getMetadataManufacturer()); } + @Test + public void testUpdateVmMetadataManufacturerAndProductCustomManufacturerDefaultProduct() { + String manufacturer = "Custom"; + overrideVmMetadataConfigValue(manufacturer, ""); + Pair pair = getDummyVmTOAndVm(); + VirtualMachineTO to = pair.first(); + virtualMachineManagerImpl.updateVmMetadataManufacturerAndProduct(to, pair.second()); + Assert.assertEquals(manufacturer, to.getMetadataManufacturer()); + Assert.assertEquals("CloudStack KVM Hypervisor", to.getMetadataProductName()); + } + @Test public void testUpdateVmMetadataManufacturerAndProductCustomManufacturer() { String manufacturer = UUID.randomUUID().toString(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index c1ea3e99717..dfd531b1a0b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -127,6 +127,9 @@ public class LibvirtVMDef { } public String getManufacturer() { + if (StringUtils.isEmpty(manufacturer)) { + return "Apache Software Foundation"; + } return manufacturer; } @@ -135,6 +138,9 @@ public class LibvirtVMDef { } public String getProduct() { + if (StringUtils.isEmpty(product)) { + return "CloudStack KVM Hypervisor"; + } return product; } From f9c4edc66fa5492909037438ff4abdb505f03a7d Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 4 Sep 2024 11:42:41 +0530 Subject: [PATCH 09/45] SystemVM template changes - updated debian version & other changes (#9625) * systemvm template changes, to explicitly update the module dependencies, vmware hardware version updated to 13, fsck added in grub * Update systemvm template debian version to 12.7 --- systemvm/debian/opt/cloud/bin/setup/init.sh | 2 ++ tools/appliance/build.sh | 2 +- tools/appliance/systemvmtemplate/scripts/configure_grub.sh | 2 +- .../template-base_aarch64-target_aarch64.json | 4 ++-- .../systemvmtemplate/template-base_x86_64-target_aarch64.json | 4 ++-- .../systemvmtemplate/template-base_x86_64-target_x86_64.json | 4 ++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/setup/init.sh b/systemvm/debian/opt/cloud/bin/setup/init.sh index ae13700fffd..5507193c4df 100644 --- a/systemvm/debian/opt/cloud/bin/setup/init.sh +++ b/systemvm/debian/opt/cloud/bin/setup/init.sh @@ -112,6 +112,8 @@ config_guest() { fi ;; vmware) + # explicitly update the module dependencies + depmod -a # system time sync'd with host via vmware tools systemctl stop ntpd systemctl disable ntpd diff --git a/tools/appliance/build.sh b/tools/appliance/build.sh index 4865842d17b..fa5d0e853cc 100755 --- a/tools/appliance/build.sh +++ b/tools/appliance/build.sh @@ -235,7 +235,7 @@ function stage_vmx() { displayname = "${1}" annotation = "${1}" guestos = "otherlinux-64" -virtualHW.version = "11" +virtualHW.version = "13" config.version = "8" numvcpus = "1" cpuid.coresPerSocket = "1" diff --git a/tools/appliance/systemvmtemplate/scripts/configure_grub.sh b/tools/appliance/systemvmtemplate/scripts/configure_grub.sh index 231aa764449..f9103925b3b 100644 --- a/tools/appliance/systemvmtemplate/scripts/configure_grub.sh +++ b/tools/appliance/systemvmtemplate/scripts/configure_grub.sh @@ -34,7 +34,7 @@ function configure_grub() { GRUB_DEFAULT=0 GRUB_TIMEOUT=0 GRUB_DISTRIBUTOR=Debian -GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX_DEFAULT="quiet fsck.mode=force fsck.repair=yes" GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200n8 console=hvc0 earlyprintk=xen net.ifnames=0 biosdevname=0 debian-installer=en_US nomodeset" GRUB_CMDLINE_XEN="com1=115200 console=com1" GRUB_TERMINAL="console serial" diff --git a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json index 2ceadeaf742..67493c7c635 100644 --- a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json @@ -32,8 +32,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:14c2ca243ee7f6e447cc4466296d974ee36645c06d72043236c3fbea78f1948d3af88d65139105a475288f270e4b636e6885143d01bdf69462620d1825e470ae", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.5.0/arm64/iso-cd/debian-12.5.0-arm64-netinst.iso", + "iso_checksum": "sha512:fc3560bb586af14b1d77ab7c2806616916926afcbd5cb3fd5a04a5633dfd91cfbbccada1a123f1ea14c480153b731cbee72a230cea17fd9116b9df8444d8df1c", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.7.0/arm64/iso-cd/debian-12.7.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json index b49ed17104a..ed03fd74942 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json @@ -31,8 +31,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:14c2ca243ee7f6e447cc4466296d974ee36645c06d72043236c3fbea78f1948d3af88d65139105a475288f270e4b636e6885143d01bdf69462620d1825e470ae", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.5.0/arm64/iso-cd/debian-12.5.0-arm64-netinst.iso", + "iso_checksum": "sha512:fc3560bb586af14b1d77ab7c2806616916926afcbd5cb3fd5a04a5633dfd91cfbbccada1a123f1ea14c480153b731cbee72a230cea17fd9116b9df8444d8df1c", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.7.0/arm64/iso-cd/debian-12.7.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json index 5dc2051d127..e209d480334 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:33c08e56c83d13007e4a5511b9bf2c4926c4aa12fd5dd56d493c0653aecbab380988c5bf1671dbaea75c582827797d98c4a611f7fb2b131fbde2c677d5258ec9", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.5.0/amd64/iso-cd/debian-12.5.0-amd64-netinst.iso", + "iso_checksum": "sha512:e0bd9ba03084a6fd42413b425a2d20e3731678a31fe5fb2cc84f79332129afca2ad4ec897b4224d6a833afaf28a5d938b0fe5d680983182944162c6825b135ce", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.7.0/amd64/iso-cd/debian-12.7.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ From 628aba618b5b7c938c494c9a7f19463d0ecce0c9 Mon Sep 17 00:00:00 2001 From: Gabriel Pordeus Santos Date: Wed, 4 Sep 2024 03:13:25 -0300 Subject: [PATCH 10/45] add min details to search view (#9616) --- ui/src/components/view/SearchView.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue index 07e2d14e8bc..0d18ab9a85c 100644 --- a/ui/src/components/view/SearchView.vue +++ b/ui/src/components/view/SearchView.vue @@ -706,7 +706,7 @@ export default { }, fetchDomains (searchKeyword) { return new Promise((resolve, reject) => { - api('listDomains', { listAll: true, showicon: true, keyword: searchKeyword }).then(json => { + api('listDomains', { listAll: true, details: 'min', showicon: true, keyword: searchKeyword }).then(json => { const domain = json.listdomainsresponse.domain resolve({ type: 'domainid', From b78aede2b705041636ffda99a25697b6f707ba43 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Wed, 4 Sep 2024 11:54:33 +0530 Subject: [PATCH 11/45] Updated listStoragePools response - added new managed parameter (#9588) --- .../api/response/StoragePoolResponse.java | 12 ++++++++++++ .../cloud/api/query/dao/StoragePoolJoinDaoImpl.java | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index bd468a9201f..06d5103d731 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -141,6 +141,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "the storage pool capabilities") private Map caps; + @SerializedName(ApiConstants.MANAGED) + @Param(description = "whether this pool is managed or not") + private Boolean managed; + public Map getCaps() { return caps; } @@ -383,4 +387,12 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { public void setTagARule(Boolean tagARule) { isTagARule = tagARule; } + + public Boolean getManaged() { + return managed; + } + + public void setManaged(Boolean managed) { + this.managed = managed; + } } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index 8c828ba2067..186e121b62c 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -171,6 +171,7 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase Date: Wed, 4 Sep 2024 11:56:17 +0530 Subject: [PATCH 12/45] server: fix volume migration check for local volume attach on a stopped (#9578) vm Fixes #8645 When a local storage volume is being attached to a stopped VM, volume migration is only needed when it is not present on the last host as the current host ID will be null in the database. Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/storage/VolumeApiServiceImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index e6092223f01..f52cd155142 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -4312,7 +4312,11 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } } else if (storeForNewStoreScope.getScopeType() == ScopeType.HOST && (storeForExistingStoreScope.getScopeType() == ScopeType.CLUSTER || storeForExistingStoreScope.getScopeType() == ScopeType.ZONE)) { - Long hostId = _vmInstanceDao.findById(existingVolume.getInstanceId()).getHostId(); + VMInstanceVO vm = _vmInstanceDao.findById(existingVolume.getInstanceId()); + Long hostId = vm.getHostId(); + if (hostId == null) { + hostId = vm.getLastHostId(); + } if (storeForNewStoreScope.getScopeId().equals(hostId)) { return false; } From 1ca9a10912d3a7014bb355a015cdf440f837b1ba Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 4 Sep 2024 08:27:28 +0200 Subject: [PATCH 13/45] VR: remove vpn user info when apply vpn users list (#9568) Prior to this PR ``` root@r-663-VM:/var/cache/cloud# gzip -dk vpn_user_list.json.aae73e2c-32ba-44f3-bf47-426933a67bcb.gz root@r-663-VM:/var/cache/cloud# /opt/cloud/bin/update_config.py vpn_user_list.json.aae73e2c-32ba-44f3-bf47-426933a67bcb {'id': 'vpnuserlist', 'test': {'add': True, 'password': 'test', 'user': 'test'}} {'vpn_users': [{'user': 'test', 'password': 'test', 'add': True}], 'type': 'vpnuserlist', 'delete_from_processed_cache': False} line = # Secrets for authentication using CHAP line = # client server secret IP addresses line = line = line = test * test * ``` with this PR ``` root@r-663-VM:/var/cache/cloud# gzip -dk vpn_user_list.json.aae73e2c-32ba-44f3-bf47-426933a67bcb.gz root@r-663-VM:/var/cache/cloud# /opt/cloud/bin/update_config.py vpn_user_list.json.aae73e2c-32ba-44f3-bf47-426933a67bcb root@r-663-VM:/var/cache/cloud# ``` --- systemvm/debian/opt/cloud/bin/cs/CsFile.py | 1 - systemvm/debian/opt/cloud/bin/cs_vpnusers.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsFile.py b/systemvm/debian/opt/cloud/bin/cs/CsFile.py index 2ee631a89d6..a60b6c6ad2e 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsFile.py @@ -153,7 +153,6 @@ class CsFile: logging.debug("Searching for %s string " % search) for index, line in enumerate(self.new_config): - print ' line = ' + line if line.lstrip().startswith(ignoreLinesStartWith): continue if search in line: diff --git a/systemvm/debian/opt/cloud/bin/cs_vpnusers.py b/systemvm/debian/opt/cloud/bin/cs_vpnusers.py index 3bef1fec239..7188741d518 100755 --- a/systemvm/debian/opt/cloud/bin/cs_vpnusers.py +++ b/systemvm/debian/opt/cloud/bin/cs_vpnusers.py @@ -22,8 +22,6 @@ import copy def merge(dbag, data): dbagc = copy.deepcopy(dbag) - print dbag - print data if "vpn_users" not in data: return dbagc From 0ba9a292d50613cb2ade2fa999d97a7cddb5f385 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Wed, 4 Sep 2024 11:58:44 +0530 Subject: [PATCH 14/45] Add validation for secstorage.allowed.internal.sites (#9567) * Add validation for secstorage.allowed.internal.sites * Address comments * Apply suggestions from code review Co-authored-by: Suresh Kumar Anaparti * Address comments --------- Co-authored-by: Suresh Kumar Anaparti --- .../configuration/ConfigurationManagerImpl.java | 14 ++++++++++++++ .../SecondaryStorageManagerImpl.java | 5 +++++ ui/src/views/setting/ConfigurationValue.vue | 2 +- .../main/java/com/cloud/utils/net/NetUtils.java | 12 ++++++++++++ .../java/com/cloud/utils/net/NetUtilsTest.java | 11 +++++++++++ 5 files changed, 43 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 29579759c7f..9df33b47257 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -303,6 +303,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Sets; import com.googlecode.ipv6.IPv6Network; +import static com.cloud.configuration.Config.SecStorageAllowedInternalDownloadSites; + public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable { public static final Logger s_logger = Logger.getLogger(ConfigurationManagerImpl.class); public static final String PERACCOUNT = "peraccount"; @@ -1314,6 +1316,18 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + if (type.equals(String.class)) { + if (name.equalsIgnoreCase(SecStorageAllowedInternalDownloadSites.key()) && StringUtils.isNotEmpty(value)) { + final String[] cidrs = value.split(","); + for (final String cidr : cidrs) { + if (!NetUtils.isValidIp4(cidr) && !NetUtils.isValidIp6(cidr) && !NetUtils.getCleanIp4Cidr(cidr).equals(cidr)) { + s_logger.error(String.format("Invalid CIDR %s value specified for the config %s", cidr, name)); + throw new InvalidParameterValueException(String.format("Invalid CIDR %s value specified for the config %s", cidr, name)); + } + } + } + } + if (configuration == null ) { //range validation has to be done per case basis, for now //return in case of Configkey parameters diff --git a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java index f37caa712bc..cd6f23923e1 100644 --- a/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java +++ b/services/secondary-storage/controller/src/main/java/org/apache/cloudstack/secondarystorage/SecondaryStorageManagerImpl.java @@ -158,6 +158,8 @@ import com.cloud.vm.dao.SecondaryStorageVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; +import static com.cloud.configuration.Config.SecStorageAllowedInternalDownloadSites; + /** * Class to manage secondary storages.

* Possible secondary storage VM state transition cases:
@@ -401,6 +403,9 @@ public class SecondaryStorageManagerImpl extends ManagerBase implements Secondar String[] cidrs = _allowedInternalSites.split(","); for (String cidr : cidrs) { if (NetUtils.isValidIp4Cidr(cidr) || NetUtils.isValidIp4(cidr) || !cidr.startsWith("0.0.0.0")) { + if (NetUtils.getCleanIp4Cidr(cidr).equals(cidr)) { + s_logger.warn(String.format("Invalid CIDR %s in %s", cidr, SecStorageAllowedInternalDownloadSites.key())); + } allowedCidrs.add(cidr); } } diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue index 0069896f7a5..836aed69dd3 100644 --- a/ui/src/views/setting/ConfigurationValue.vue +++ b/ui/src/views/setting/ConfigurationValue.vue @@ -266,7 +266,7 @@ export default { this.$message.error(this.$t('message.error.save.setting')) this.$notification.error({ message: this.$t('label.error'), - description: this.$t('message.error.save.setting') + description: error?.response?.data?.updateconfigurationresponse?.errortext || this.$t('message.error.save.setting') }) }).finally(() => { this.valueLoading = false diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index 91a2f4eb755..1b4ebcccf94 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -626,6 +626,18 @@ public class NetUtils { return long2Ip(firstPart) + "/" + size; } + public static String getCleanIp4Cidr(final String cidr) { + if (!isValidIp4Cidr(cidr)) { + throw new CloudRuntimeException("Invalid CIDR: " + cidr); + } + String gateway = cidr.split("/")[0]; + Long netmaskSize = Long.parseLong(cidr.split("/")[1]); + final long ip = ip2Long(gateway); + final long startNetMask = ip2Long(getCidrNetmask(netmaskSize)); + final long start = (ip & startNetMask); + return String.format("%s/%s", long2Ip(start), netmaskSize); + } + public static String[] getIpRangeFromCidr(final String cidr, final long size) { assert size < MAX_CIDR : "You do know this is not for ipv6 right? Keep it smaller than 32 but you have " + size; final String[] result = new String[2]; diff --git a/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java b/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java index defb440c2a5..0f19da38922 100644 --- a/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java +++ b/utils/src/test/java/com/cloud/utils/net/NetUtilsTest.java @@ -296,6 +296,17 @@ public class NetUtilsTest { assertTrue(NetUtils.isValidIp4Cidr(cidrThird));; } + @Test + public void testGetCleanIp4Cidr() throws Exception { + final String cidrFirst = "10.0.144.0/20"; + final String cidrSecond = "10.0.151.5/20"; + final String cidrThird = "10.0.144.10/21"; + + assertEquals(cidrFirst, NetUtils.getCleanIp4Cidr(cidrFirst)); + assertEquals("10.0.144.0/20", NetUtils.getCleanIp4Cidr(cidrSecond)); + assertEquals("10.0.144.0/21", NetUtils.getCleanIp4Cidr(cidrThird));; + } + @Test public void testIsValidCidrList() throws Exception { final String cidrFirst = "10.0.144.0/20,1.2.3.4/32,5.6.7.8/24"; From e06f80e899127abe45c10ff2d38420a770ceb076 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 4 Sep 2024 08:32:03 +0200 Subject: [PATCH 15/45] storage: fix private templates are not copied to new image store (#9206) ``` 2024-06-10T12:24:50,711 INFO [o.a.c.s.i.TemplateServiceImpl] (qtp776484396-20:[ctx-eb090c22, ctx-5fa5579c]) (logid:7a783000) Template Sync did not find 211-2-d536fb03-5f89-3e77-8dea-323315bcbfab on image store 3, may request download based on available hypervisor types ... 2024-06-10T12:24:51,043 INFO [o.a.c.s.i.TemplateServiceImpl] (qtp776484396-20:[ctx-eb090c22, ctx-5fa5579c]) (logid:7a783000) Skip sync downloading private template 211-2-d536fb03-5f89-3e77-8dea-323315bcbfab to a new image store ``` --- .../apache/cloudstack/storage/image/TemplateServiceImpl.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index 6c4fcab2f17..c040e1a7ad6 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -532,11 +532,6 @@ public class TemplateServiceImpl implements TemplateService { s_logger.info("Skip downloading template " + tmplt.getUniqueName() + " since no url is specified."); continue; } - // if this is private template, skip sync to a new image store - if (isSkipTemplateStoreDownload(tmplt, zoneId)) { - s_logger.info("Skip sync downloading private template " + tmplt.getUniqueName() + " to a new image store"); - continue; - } // if this is a region store, and there is already an DOWNLOADED entry there without install_path information, which // means that this is a duplicate entry from migration of previous NFS to staging. From 882dea21c1fc584255bf7214a7b201a52bb9236d Mon Sep 17 00:00:00 2001 From: Felipe <124818914+FelipeM525@users.noreply.github.com> Date: Wed, 4 Sep 2024 03:32:28 -0300 Subject: [PATCH 16/45] Update .asf.yaml (#9629) --- .asf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.asf.yaml b/.asf.yaml index 8c1a5d51fdf..4d979a18833 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -59,6 +59,7 @@ github: - hsato03 - bernardodemarco - abh1sar + - FelipeM525 protected_branches: ~ From f2a1ee57cac18904d798b400b182fa04b814cff7 Mon Sep 17 00:00:00 2001 From: NuxRo Date: Wed, 4 Sep 2024 07:34:55 +0100 Subject: [PATCH 17/45] Update en.json (#8958) Replace "Control Plane" with "Compute Resource" --- ui/public/locales/en.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 2e5412bf737..c87ee0070f7 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1036,7 +1036,7 @@ "label.host.alerts": "Hosts in alert state", "label.host.name": "Host name", "label.host.tag": "Host tag", -"label.hostcontrolstate": "Control Plane Status", +"label.hostcontrolstate": "Compute Resource Status", "label.hostid": "Host", "label.hostname": "Host", "label.hostname.tooltip": "Destination Host. Volume should be located in local storage of this Host", @@ -3079,7 +3079,7 @@ "message.chart.statistic.info.hypervisor.additionals": "The metrics data depend on the hypervisor plugin used for each hypervisor. The behavior can vary across different hypervisors. For instance, with KVM, metrics are real-time statistics provided by libvirt. In contrast, with VMware, the metrics are averaged data for a given time interval controlled by configuration.", "message.guest.traffic.in.advanced.zone": "Guest Network traffic is communication between end-user Instances. Specify a range of VLAN IDs or VXLAN Network identifiers (VNIs) to carry guest traffic for each physical Network.", "message.guest.traffic.in.basic.zone": "Guest Network traffic is communication between end-user Instances. Specify a range of IP addresses that CloudStack can assign to guest Instances. Make sure this range does not overlap the reserved system IP range.", -"message.host.controlstate": "The Control Plane Status of this Instance is ", +"message.host.controlstate": "The Compute Resource Status for this Instance is ", "message.host.controlstate.retry": "Some actions on this Instance will fail, if so please wait a while and retry.", "message.host.dedicated": "Host Dedicated", "message.host.dedication.released": "Host dedication released.", From 24dc3178a39d5cc4833539264abae07c1a618683 Mon Sep 17 00:00:00 2001 From: Alain-Christian Courtines <110478358+my-code-AL@users.noreply.github.com> Date: Tue, 3 Sep 2024 23:39:47 -0700 Subject: [PATCH 18/45] Testcases Added (#9116) * added a news tester file and directory for GsonHelper.java. Also created a new test in OVAProcessorTest.java * added testcase to PasswordPolicyImplTest.java * added proper imports for GsonHelperTest.java * expected changed based on commit response * adhere to checkstyle ruleset --- .../com/cloud/serializer/GsonHelperTest.java | 81 +++++++++++++++++++ .../storage/template/OVAProcessorTest.java | 20 +++++ .../cloud/user/PasswordPolicyImplTest.java | 18 +++++ 3 files changed, 119 insertions(+) create mode 100644 core/src/test/java/com/cloud/serializer/GsonHelperTest.java diff --git a/core/src/test/java/com/cloud/serializer/GsonHelperTest.java b/core/src/test/java/com/cloud/serializer/GsonHelperTest.java new file mode 100644 index 00000000000..e8b0b373060 --- /dev/null +++ b/core/src/test/java/com/cloud/serializer/GsonHelperTest.java @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.serializer; + +import com.cloud.agent.api.to.NfsTO; +import com.cloud.storage.DataStoreRole; +import com.google.gson.Gson; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test cases to verify working order of GsonHelper.java + * with regards to a concrete implementation of the DataStoreTO + * interface + */ +public class GsonHelperTest { + + private Gson gson; + private Gson gsonLogger; + private NfsTO nfsTO; + + @Before + public void setUp() { + gson = GsonHelper.getGson(); + gsonLogger = GsonHelper.getGsonLogger(); + nfsTO = new NfsTO("http://example.com", DataStoreRole.Primary); + } + + @Test + public void testGsonSerialization() { + String json = gson.toJson(nfsTO); + assertNotNull(json); + assertTrue(json.contains("\"_url\":\"http://example.com\"")); + assertTrue(json.contains("\"_role\":\"Primary\"")); + } + + @Test + public void testGsonDeserialization() { + String json = "{\"_url\":\"http://example.com\",\"_role\":\"Primary\"}"; + NfsTO deserializedNfsTO = gson.fromJson(json, NfsTO.class); + assertNotNull(deserializedNfsTO); + assertEquals("http://example.com", deserializedNfsTO.getUrl()); + assertEquals(DataStoreRole.Primary, deserializedNfsTO.getRole()); + } + + @Test + public void testGsonLoggerSerialization() { + String json = gsonLogger.toJson(nfsTO); + assertNotNull(json); + assertTrue(json.contains("\"_url\":\"http://example.com\"")); + assertTrue(json.contains("\"_role\":\"Primary\"")); + } + + @Test + public void testGsonLoggerDeserialization() { + String json ="{\"_url\":\"http://example.com\",\"_role\":\"Primary\"}"; + NfsTO deserializedNfsTO = gsonLogger.fromJson(json, NfsTO.class); + assertNotNull(deserializedNfsTO); + assertEquals("http://example.com", deserializedNfsTO.getUrl()); + assertEquals(DataStoreRole.Primary, deserializedNfsTO.getRole()); + } +} diff --git a/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java b/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java index 8ab54644718..8674a8df286 100644 --- a/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java +++ b/core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java @@ -131,5 +131,25 @@ public class OVAProcessorTest { Assert.assertEquals(virtualSize, processor.getVirtualSize(mockFile)); Mockito.verify(mockFile, Mockito.times(0)).length(); } + @Test + public void testProcessWithLargeFileSize() throws Exception { + String templatePath = "/tmp"; + String templateName = "large_template"; + long virtualSize = 10_000_000_000L; + long actualSize = 5_000_000_000L; + Mockito.when(mockStorageLayer.exists(Mockito.anyString())).thenReturn(true); + Mockito.when(mockStorageLayer.getSize(Mockito.anyString())).thenReturn(actualSize); + Mockito.doReturn(virtualSize).when(processor).getTemplateVirtualSize(Mockito.anyString(), Mockito.anyString()); + + try (MockedConstruction diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 27bd6a2fb36..2f94d6436a4 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -90,6 +90,13 @@ {{ text }} {{ text }} + +   + + + + + diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue index 0d18ab9a85c..8d702eafdcd 100644 --- a/ui/src/components/view/SearchView.vue +++ b/ui/src/components/view/SearchView.vue @@ -303,7 +303,7 @@ export default { } if (['zoneid', 'domainid', 'imagestoreid', 'storageid', 'state', 'account', 'hypervisor', 'level', 'clusterid', 'podid', 'groupid', 'entitytype', 'accounttype', 'systemvmtype', 'scope', 'provider', - 'type', 'scope', 'managementserverid', 'serviceofferingid', 'diskofferingid', 'usagetype'].includes(item) + 'type', 'scope', 'managementserverid', 'serviceofferingid', 'diskofferingid', 'usagetype', 'restartrequired'].includes(item) ) { type = 'list' } else if (item === 'tags') { @@ -395,6 +395,16 @@ export default { this.fields[providerIndex].loading = false } + if (arrayField.includes('restartrequired')) { + const restartRequiredIndex = this.fields.findIndex(item => item.name === 'restartrequired') + this.fields[restartRequiredIndex].loading = true + this.fields[restartRequiredIndex].opts = [ + { id: 'true', name: 'label.yes' }, + { id: 'false', name: 'label.no' } + ] + this.fields[restartRequiredIndex].loading = false + } + if (arrayField.includes('resourcetype')) { const resourceTypeIndex = this.fields.findIndex(item => item.name === 'resourcetype') this.fields[resourceTypeIndex].loading = true diff --git a/ui/src/config/section/network.js b/ui/src/config/section/network.js index 26b4f279e3d..1ff704a306f 100644 --- a/ui/src/config/section/network.js +++ b/ui/src/config/section/network.js @@ -54,7 +54,7 @@ export default { return fields }, filters: ['all', 'account', 'domainpath', 'shared'], - searchFilters: ['keyword', 'zoneid', 'domainid', 'account', 'type', 'tags'], + searchFilters: ['keyword', 'zoneid', 'domainid', 'account', 'type', 'restartrequired', 'tags'], related: [{ name: 'vm', title: 'label.instances', @@ -218,7 +218,7 @@ export default { return fields }, details: ['name', 'id', 'displaytext', 'cidr', 'networkdomain', 'ip6routes', 'ispersistent', 'redundantvpcrouter', 'restartrequired', 'zonename', 'account', 'domain', 'dns1', 'dns2', 'ip6dns1', 'ip6dns2', 'publicmtu'], - searchFilters: ['name', 'zoneid', 'domainid', 'account', 'tags'], + searchFilters: ['name', 'zoneid', 'domainid', 'account', 'restartrequired', 'tags'], related: [{ name: 'vm', title: 'label.instances', diff --git a/ui/src/store/getters.js b/ui/src/store/getters.js index 67b168be8c2..405fd11bad1 100644 --- a/ui/src/store/getters.js +++ b/ui/src/store/getters.js @@ -28,6 +28,7 @@ const getters = { apis: state => state.user.apis, features: state => state.user.features, userInfo: state => state.user.info, + latestVersion: state => state.user.latestVersion, addRouters: state => state.permission.addRouters, multiTab: state => state.app.multiTab, listAllProjects: state => state.app.listAllProjects, diff --git a/ui/src/store/modules/user.js b/ui/src/store/modules/user.js index 24302a94033..a5b48acf99d 100644 --- a/ui/src/store/modules/user.js +++ b/ui/src/store/modules/user.js @@ -18,12 +18,15 @@ import Cookies from 'js-cookie' import message from 'ant-design-vue/es/message' import notification from 'ant-design-vue/es/notification' +import semver from 'semver' import { vueProps } from '@/vue-app' import router from '@/router' import store from '@/store' import { oauthlogin, login, logout, api } from '@/api' import { i18n } from '@/locales' +import { axios } from '../../utils/request' +import { getParsedVersion } from '@/utils/util' import { ACCESS_TOKEN, @@ -38,7 +41,8 @@ import { DARK_MODE, CUSTOM_COLUMNS, OAUTH_DOMAIN, - OAUTH_PROVIDER + OAUTH_PROVIDER, + LATEST_CS_VERSION } from '@/store/mutation-types' const user = { @@ -167,6 +171,12 @@ const user = { }, SET_OAUTH_PROVIDER_USED_TO_LOGIN: (state, provider) => { vueProps.$localStorage.set(OAUTH_PROVIDER, provider) + }, + SET_LATEST_VERSION: (state, version) => { + if (version?.fetchedTs > 0) { + vueProps.$localStorage.set(LATEST_CS_VERSION, version) + state.latestVersion = version + } } }, @@ -212,6 +222,8 @@ const user = { commit('SET_2FA_PROVIDER', result.providerfor2fa) commit('SET_2FA_ISSUER', result.issuerfor2fa) commit('SET_LOGIN_FLAG', false) + const latestVersion = vueProps.$localStorage.get(LATEST_CS_VERSION, { version: '', fetchedTs: 0 }) + commit('SET_LATEST_VERSION', latestVersion) notification.destroy() resolve() @@ -259,6 +271,8 @@ const user = { commit('SET_2FA_PROVIDER', result.providerfor2fa) commit('SET_2FA_ISSUER', result.issuerfor2fa) commit('SET_LOGIN_FLAG', false) + const latestVersion = vueProps.$localStorage.get(LATEST_CS_VERSION, { version: '', fetchedTs: 0 }) + commit('SET_LATEST_VERSION', latestVersion) notification.destroy() resolve() @@ -277,10 +291,12 @@ const user = { const cachedCustomColumns = vueProps.$localStorage.get(CUSTOM_COLUMNS, {}) const domainStore = vueProps.$localStorage.get(DOMAIN_STORE, {}) const darkMode = vueProps.$localStorage.get(DARK_MODE, false) + const latestVersion = vueProps.$localStorage.get(LATEST_CS_VERSION, { version: '', fetchedTs: 0 }) const hasAuth = Object.keys(cachedApis).length > 0 commit('SET_DOMAIN_STORE', domainStore) commit('SET_DARK_MODE', darkMode) + commit('SET_LATEST_VERSION', latestVersion) if (hasAuth) { console.log('Login detected, using cached APIs') commit('SET_ZONES', cachedZones) @@ -294,6 +310,7 @@ const user = { const result = response.listusersresponse.user[0] commit('SET_INFO', result) commit('SET_NAME', result.firstname + ' ' + result.lastname) + store.dispatch('SetCsLatestVersion', result.rolename) resolve(cachedApis) }).catch(error => { reject(error) @@ -332,12 +349,41 @@ const user = { }).catch(error => { reject(error) }) + + api('listNetworks', { restartrequired: true, forvpc: false }).then(response => { + if (response.listnetworksresponse.count > 0) { + store.dispatch('AddHeaderNotice', { + key: 'NETWORK_RESTART_REQUIRED', + title: i18n.global.t('label.network.restart.required'), + description: i18n.global.t('message.network.restart.required'), + path: '/guestnetwork/', + query: { restartrequired: true, forvpc: false }, + status: 'done', + timestamp: new Date() + }) + } + }).catch(ignored => {}) + + api('listVPCs', { restartrequired: true }).then(response => { + if (response.listvpcsresponse.count > 0) { + store.dispatch('AddHeaderNotice', { + key: 'VPC_RESTART_REQUIRED', + title: i18n.global.t('label.vpc.restart.required'), + description: i18n.global.t('message.vpc.restart.required'), + path: '/vpc/', + query: { restartrequired: true }, + status: 'done', + timestamp: new Date() + }) + } + }).catch(ignored => {}) } api('listUsers', { username: Cookies.get('username') }).then(response => { const result = response.listusersresponse.user[0] commit('SET_INFO', result) commit('SET_NAME', result.firstname + ' ' + result.lastname) + store.dispatch('SetCsLatestVersion', result.rolename) }).catch(error => { reject(error) }) @@ -367,6 +413,8 @@ const user = { commit('SET_CLOUDIAN', cloudian) }).catch(ignored => { }) + }).catch(error => { + console.error(error) }) }, @@ -488,6 +536,29 @@ const user = { SetDomainStore ({ commit }, domainStore) { commit('SET_DOMAIN_STORE', domainStore) }, + SetCsLatestVersion ({ commit }, rolename) { + const lastFetchTs = store.getters.latestVersion?.fetchedTs ? store.getters.latestVersion.fetchedTs : 0 + if (rolename === 'Root Admin' && (+new Date() - lastFetchTs) > 24 * 60 * 60 * 1000) { + axios.get( + 'https://api.github.com/repos/apache/cloudstack/releases' + ).then(response => { + let latestReleaseVersion = getParsedVersion(response[0].tag_name) + let latestTag = response[0].tag_name + + for (const release of response) { + if (release.tag_name.toLowerCase().includes('rc')) { + continue + } + const parsedVersion = getParsedVersion(release.tag_name) + if (semver.gte(parsedVersion, latestReleaseVersion)) { + latestReleaseVersion = parsedVersion + latestTag = release.tag_name + commit('SET_LATEST_VERSION', { version: latestTag, fetchedTs: (+new Date()) }) + } + } + }).catch(ignored => {}) + } + }, SetDarkMode ({ commit }, darkMode) { commit('SET_DARK_MODE', darkMode) }, diff --git a/ui/src/store/mutation-types.js b/ui/src/store/mutation-types.js index 77aeb8fb7b6..93dfc9fbc20 100644 --- a/ui/src/store/mutation-types.js +++ b/ui/src/store/mutation-types.js @@ -35,6 +35,7 @@ export const USE_BROWSER_TIMEZONE = 'USE_BROWSER_TIMEZONE' export const SERVER_MANAGER = 'SERVER_MANAGER' export const DOMAIN_STORE = 'DOMAIN_STORE' export const DARK_MODE = 'DARK_MODE' +export const LATEST_CS_VERSION = 'LATEST_CS_VERSION' export const VUE_VERSION = 'VUE_VERSION' export const CUSTOM_COLUMNS = 'CUSTOM_COLUMNS' export const RELOAD_ALL_PROJECTS = 'RELOAD_ALL_PROJECTS' diff --git a/ui/src/utils/util.js b/ui/src/utils/util.js index fad4d1e0f5d..8773f07446e 100644 --- a/ui/src/utils/util.js +++ b/ui/src/utils/util.js @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import semver from 'semver' + export function timeFix () { const time = new Date() const hour = time.getHours() @@ -69,6 +71,14 @@ export function sanitizeReverse (value) { .replace(/>/g, '>') } +export function getParsedVersion (version) { + version = version.split('-')[0] + if (semver.valid(version) === null) { + version = version.split('.').slice(1).join('.') + } + return version +} + export function toCsv ({ keys = null, data = null, columnDelimiter = ',', lineDelimiter = '\n' }) { if (data === null || !data.length) { return null From bc393923515f9795f3f983ba4040e3f0a5326efe Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 4 Sep 2024 09:45:39 +0200 Subject: [PATCH 23/45] Fix PR lint error caused by deps/install-non-oss.sh (#9631) --- deps/install-non-oss.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/deps/install-non-oss.sh b/deps/install-non-oss.sh index f571af239bd..56cfc2cba9c 100755 --- a/deps/install-non-oss.sh +++ b/deps/install-non-oss.sh @@ -86,4 +86,3 @@ mvn install:install-file -Dfile=juniper-contrail-api-1.0-SNAPSHOT.jar -DgroupId= mvn install:install-file -Dfile=juniper-tungsten-api-2.0.jar -DgroupId=net.juniper.tungsten -DartifactId=juniper-tungsten-api -Dversion=2.0 -Dpackaging=jar exit 0 - From 97c1a86b646e0373adfd741bf09be38bd7d6ffd4 Mon Sep 17 00:00:00 2001 From: Rene Peinthor Date: Wed, 4 Sep 2024 11:45:56 +0200 Subject: [PATCH 24/45] linstor: update java-linstor dependency to 0.5.2 (#9632) 0.5.1 had a bug while retrieving the bestErrorMessage for an apicall answer. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fc26ca957c0..da0252acbb4 100644 --- a/pom.xml +++ b/pom.xml @@ -169,7 +169,7 @@ 10.1 2.6.6 0.6.0 - 0.5.1 + 0.5.2 0.10.2 3.4.4_1 4.0.1 From 5ff0b999da206862a9108918726c70227fd868d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 19:14:40 +0530 Subject: [PATCH 25/45] Bump org.apache.commons:commons-compress from 1.21 to 1.26.0 (#8683) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ed3fa73506e..32811897fa1 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,7 @@ 1.15 1.5.0 4.4 - 1.21 + 1.26.0 1.3 1.4 3.1 From dda9ef2dc83645686e44d8cdfeb7cebe5ac78376 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 4 Sep 2024 18:44:19 +0200 Subject: [PATCH 26/45] UI: list vms with details=min when attach a volume to vm (#9634) --- ui/src/views/storage/AttachVolume.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/src/views/storage/AttachVolume.vue b/ui/src/views/storage/AttachVolume.vue index de0a2f7e80c..e0aa59a0be6 100644 --- a/ui/src/views/storage/AttachVolume.vue +++ b/ui/src/views/storage/AttachVolume.vue @@ -110,7 +110,8 @@ export default { }, fetchData () { var params = { - zoneid: this.resource.zoneid + zoneid: this.resource.zoneid, + details: 'min' } if (this.resource.hypervisor && this.resource.hypervisor !== 'None') { params.hypervisor = this.resource.hypervisor From 787acfd172542f24460318bf7b83ca77ba96484f Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Thu, 5 Sep 2024 09:29:00 +0530 Subject: [PATCH 27/45] vmware: Add support for VMware 8.0u2 (8.0.2.x) and 8.0u3 (8.0.3.x) (#9591) Add support for VMware 8.0u2 (8.0.2.x) and 8.0u3 (8.0.3.x) --- .../src/main/resources/META-INF/db/schema-41910to42000.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql index 9abaa2c22da..2b6682b2502 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql @@ -233,3 +233,9 @@ CALL `cloud`.`IDEMPOTENT_MODIFY_COLUMN_CHAR_SET`('vpc_offerings', 'unique_name', CALL `cloud`.`IDEMPOTENT_MODIFY_COLUMN_CHAR_SET`('vpc_offerings', 'display_text', 'VARCHAR(255)', 'DEFAULT NULL COMMENT \'display text\''); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.roles','state', 'varchar(10) NOT NULL default "enabled" COMMENT "role state"'); + +-- Add support for VMware 8.0u2 (8.0.2.x) and 8.0u3 (8.0.3.x) +INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.2', 1024, 0, 59, 64, 1, 1); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '8.0.2', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='8.0'; +INSERT IGNORE INTO `cloud`.`hypervisor_capabilities` (uuid, hypervisor_type, hypervisor_version, max_guests_limit, security_group_enabled, max_data_volumes_limit, max_hosts_per_cluster, storage_motion_supported, vm_snapshot_enabled) values (UUID(), 'VMware', '8.0.3', 1024, 0, 59, 64, 1, 1); +INSERT IGNORE INTO `cloud`.`guest_os_hypervisor` (uuid, hypervisor_type, hypervisor_version, guest_os_name, guest_os_id, created, is_user_defined) SELECT UUID(),'VMware', '8.0.3', guest_os_name, guest_os_id, utc_timestamp(), 0 FROM `cloud`.`guest_os_hypervisor` WHERE hypervisor_type='VMware' AND hypervisor_version='8.0'; From 31b0ed0a18c5aa7caa1bee0c7c4e6090eac7e588 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 5 Sep 2024 09:32:56 +0530 Subject: [PATCH 28/45] framework/config,server: configkey caching (#9628) Added caching for ConfigKey value retrievals based on the Caffeine in-memory caching library. https://github.com/ben-manes/caffeine Currently, expire time for a cache is 30s and each update of the config key invalidates the cache. On any update or reset of the configuration, cache automatically invalidates for it. Signed-off-by: Abhishek Kumar --- .../vm/VirtualMachineManagerImplTest.java | 11 +-- .../com/cloud/dc/ClusterDetailsDaoImpl.java | 5 +- .../dc/dao/DataCenterDetailsDaoImpl.java | 4 +- .../domain/dao/DomainDetailsDaoImpl.java | 16 ++-- .../dao/StoragePoolDetailsDaoImpl.java | 11 +-- .../com/cloud/user/AccountDetailsDaoImpl.java | 15 ++-- .../db/ImageStoreDetailsDaoImpl.java | 15 ++-- .../framework/config/ConfigDepot.java | 2 + .../framework/config/ConfigKey.java | 31 +++---- .../framework/config/ScopedConfigStorage.java | 6 +- .../config/impl/ConfigDepotImpl.java | 53 ++++++++++- .../config/impl/ConfigDepotImplTest.java | 50 +++++++++++ pom.xml | 6 ++ .../ConfigurationManagerImpl.java | 31 ++++--- .../com/cloud/vm/FirstFitPlannerTest.java | 90 +++++++++---------- ui/public/locales/en.json | 1 + ui/src/components/view/ListView.vue | 11 ++- ui/src/components/view/SettingsTab.vue | 10 ++- ui/src/views/setting/ConfigurationValue.vue | 12 ++- 19 files changed, 255 insertions(+), 125 deletions(-) diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index 906cded455e..47f9b9f33e2 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -43,7 +43,6 @@ import java.util.stream.Collectors; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.ScopedConfigStorage; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -1256,12 +1255,10 @@ public class VirtualMachineManagerImplTest { ConfigKey configKey = VirtualMachineManager.VmMetadataManufacturer; this.configDepotImpl = (ConfigDepotImpl)ReflectionTestUtils.getField(configKey, "s_depot"); ConfigDepotImpl configDepot = Mockito.mock(ConfigDepotImpl.class); - ScopedConfigStorage storage = Mockito.mock(ScopedConfigStorage.class); - Mockito.when(storage.getConfigValue(Mockito.anyLong(), Mockito.eq(configKey))).thenReturn(manufacturer); - Mockito.when(storage.getConfigValue(Mockito.anyLong(), Mockito.eq(VirtualMachineManager.VmMetadataProductName))) - .thenReturn(product); - Mockito.when(configDepot.findScopedConfigStorage(configKey)).thenReturn(storage); - Mockito.when(configDepot.findScopedConfigStorage(VirtualMachineManager.VmMetadataProductName)).thenReturn(storage); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(configKey.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn(manufacturer); + Mockito.when(configDepot.getConfigStringValue(Mockito.eq(VirtualMachineManager.VmMetadataProductName.key()), + Mockito.eq(ConfigKey.Scope.Zone), Mockito.anyLong())).thenReturn(product); ReflectionTestUtils.setField(configKey, "s_depot", configDepot); updatedConfigKeyDepot = true; } diff --git a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java index c2058ad5644..0e40f8475c1 100644 --- a/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/ClusterDetailsDaoImpl.java @@ -20,7 +20,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; @@ -136,8 +135,8 @@ public class ClusterDetailsDaoImpl extends GenericDaoBase key) { - ClusterDetailsVO vo = findDetail(id, key.key()); + public String getConfigValue(long id, String key) { + ClusterDetailsVO vo = findDetail(id, key); return vo == null ? null : vo.getValue(); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java index e36c8ebd6c7..bb03a96d02e 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DataCenterDetailsDaoImpl.java @@ -44,8 +44,8 @@ public class DataCenterDetailsDaoImpl extends ResourceDetailsDaoBase key) { - ResourceDetail vo = findDetail(id, key.key()); + public String getConfigValue(long id, String key) { + ResourceDetail vo = findDetail(id, key); return vo == null ? null : vo.getValue(); } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java index 50097d154f5..b9721a2e58c 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDetailsDaoImpl.java @@ -22,6 +22,11 @@ import java.util.Map; import javax.inject.Inject; +import org.apache.cloudstack.framework.config.ConfigKey.Scope; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; + import com.cloud.domain.DomainDetailVO; import com.cloud.domain.DomainVO; import com.cloud.utils.crypt.DBEncryptionUtil; @@ -31,11 +36,6 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.ConfigKey.Scope; -import org.apache.cloudstack.framework.config.ScopedConfigStorage; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; public class DomainDetailsDaoImpl extends GenericDaoBase implements DomainDetailsDao, ScopedConfigStorage { protected final SearchBuilder domainSearch; @@ -108,17 +108,17 @@ public class DomainDetailsDaoImpl extends GenericDaoBase i } @Override - public String getConfigValue(long id, ConfigKey key) { + public String getConfigValue(long id, String key) { DomainDetailVO vo = null; String enableDomainSettingsForChildDomain = _configDao.getValue("enable.domain.settings.for.child.domain"); if (!Boolean.parseBoolean(enableDomainSettingsForChildDomain)) { - vo = findDetail(id, key.key()); + vo = findDetail(id, key); return vo == null ? null : getActualValue(vo); } DomainVO domain = _domainDao.findById(id); // if value is not configured in domain then check its parent domain till ROOT while (domain != null) { - vo = findDetail(domain.getId(), key.key()); + vo = findDetail(domain.getId(), key); if (vo != null) { break; } else if (domain.getParent() != null) { diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java index 0c39a8c581a..376933f92e7 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolDetailsDaoImpl.java @@ -17,6 +17,10 @@ package com.cloud.storage.dao; +import java.util.List; + +import javax.inject.Inject; + import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; @@ -26,9 +30,6 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import javax.inject.Inject; -import java.util.List; - public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase implements StoragePoolDetailsDao, ScopedConfigStorage { @Inject @@ -43,8 +44,8 @@ public class StoragePoolDetailsDaoImpl extends ResourceDetailsDaoBase key) { - StoragePoolDetailVO vo = findDetail(id, key.key()); + public String getConfigValue(long id, String key) { + StoragePoolDetailVO vo = findDetail(id, key); return vo == null ? null : vo.getValue(); } diff --git a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java index de562e27f9e..510270ad7bf 100644 --- a/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/user/AccountDetailsDaoImpl.java @@ -23,25 +23,24 @@ import java.util.Optional; import javax.inject.Inject; -import com.cloud.utils.crypt.DBEncryptionUtil; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.ConfigKey.Scope; import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import com.cloud.domain.DomainDetailVO; import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDetailsDao; import com.cloud.domain.dao.DomainDao; +import com.cloud.domain.dao.DomainDetailsDao; import com.cloud.user.dao.AccountDao; - +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; public class AccountDetailsDaoImpl extends GenericDaoBase implements AccountDetailsDao, ScopedConfigStorage { protected final SearchBuilder accountSearch; @@ -118,9 +117,9 @@ public class AccountDetailsDaoImpl extends GenericDaoBase } @Override - public String getConfigValue(long id, ConfigKey key) { + public String getConfigValue(long id, String key) { // check if account level setting is configured - AccountDetailVO vo = findDetail(id, key.key()); + AccountDetailVO vo = findDetail(id, key); String value = vo == null ? null : getActualValue(vo); if (value != null) { return value; @@ -140,7 +139,7 @@ public class AccountDetailsDaoImpl extends GenericDaoBase if (account.isPresent()) { DomainVO domain = _domainDao.findById(account.get().getDomainId()); while (domain != null) { - DomainDetailVO domainVO = _domainDetailsDao.findDetail(domain.getId(), key.key()); + DomainDetailVO domainVO = _domainDetailsDao.findDetail(domain.getId(), key); if (domainVO != null) { value = _domainDetailsDao.getActualValue(domainVO); break; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java index 8e5ce770f45..14830490600 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDetailsDaoImpl.java @@ -20,6 +20,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ConfigKey.Scope; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; import org.springframework.stereotype.Component; import com.cloud.utils.crypt.DBEncryptionUtil; @@ -29,12 +34,6 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.ConfigKey.Scope; -import org.apache.cloudstack.framework.config.ScopedConfigStorage; -import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; - @Component public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ImageStoreDetailsDao, ScopedConfigStorage { @@ -106,8 +105,8 @@ public class ImageStoreDetailsDaoImpl extends ResourceDetailsDaoBase key) { - ImageStoreDetailVO vo = findDetail(id, key.key()); + public String getConfigValue(long id, String key) { + ImageStoreDetailVO vo = findDetail(id, key); return vo == null ? null : vo.getValue(); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java index b38b30e88b8..5ee5f9dec48 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigDepot.java @@ -32,4 +32,6 @@ public interface ConfigDepot { void createOrUpdateConfigObject(String componentName, ConfigKey key, String value); boolean isNewConfig(ConfigKey configKey); + String getConfigStringValue(String key, ConfigKey.Scope scope, Long scopeId); + void invalidateConfigCache(String key, ConfigKey.Scope scope, Long scopeId); } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java index fa570e0e8fb..36a8050754c 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ConfigKey.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.framework.config; import java.sql.Date; import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -215,42 +214,38 @@ public class ConfigKey { public T value() { if (_value == null || isDynamic()) { - ConfigurationVO vo = (s_depot != null && s_depot.global() != null) ? s_depot.global().findById(key()) : null; - final String value = (vo != null && vo.getValue() != null) ? vo.getValue() : defaultValue(); - _value = ((value == null) ? (T)defaultValue() : valueOf(value)); + String value = s_depot != null ? s_depot.getConfigStringValue(_name, Scope.Global, null) : null; + _value = valueOf((value == null) ? defaultValue() : value); } return _value; } - public T valueIn(Long id) { + protected T valueInScope(Scope scope, Long id) { if (id == null) { return value(); } - String value = s_depot != null ? s_depot.findScopedConfigStorage(this).getConfigValue(id, this) : null; + String value = s_depot != null ? s_depot.getConfigStringValue(_name, scope, id) : null; if (value == null) { return value(); - } else { - return valueOf(value); } + return valueOf(value); + } + + public T valueIn(Long id) { + return valueInScope(_scope, id); } public T valueInDomain(Long domainId) { - if (domainId == null) { - return value(); - } - - String value = s_depot != null ? s_depot.getDomainScope(this).getConfigValue(domainId, this) : null; - if (value == null) { - return value(); - } else { - return valueOf(value); - } + return valueInScope(Scope.Domain, domainId); } @SuppressWarnings("unchecked") protected T valueOf(String value) { + if (value == null) { + return null; + } Number multiplier = 1; if (multiplier() != null) { multiplier = (Number)multiplier(); diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java index f990278b45c..8126b9510a2 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/ScopedConfigStorage.java @@ -26,5 +26,9 @@ import org.apache.cloudstack.framework.config.ConfigKey.Scope; public interface ScopedConfigStorage { Scope getScope(); - String getConfigValue(long id, ConfigKey key); + String getConfigValue(long id, String key); + + default String getConfigValue(long id, ConfigKey key) { + return getConfigValue(id, key.key()); + } } diff --git a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java index 6884043cae2..b47370d9205 100644 --- a/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java +++ b/framework/config/src/main/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImpl.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.concurrent.TimeUnit; import javax.annotation.PostConstruct; import javax.inject.Inject; @@ -37,12 +38,14 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao; import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; import org.apache.commons.lang.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; import com.cloud.utils.exception.CloudRuntimeException; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; /** * ConfigDepotImpl implements the ConfigDepot and ConfigDepotAdmin interface. @@ -73,6 +76,7 @@ import com.cloud.utils.exception.CloudRuntimeException; */ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { protected Logger logger = LogManager.getLogger(getClass()); + protected final static long CONFIG_CACHE_EXPIRE_SECONDS = 30; @Inject ConfigurationDao _configDao; @Inject @@ -83,12 +87,17 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { List _scopedStorages; Set _configured = Collections.synchronizedSet(new HashSet()); Set newConfigs = Collections.synchronizedSet(new HashSet<>()); + Cache configCache; private HashMap>> _allKeys = new HashMap>>(1007); HashMap>> _scopeLevelConfigsMap = new HashMap>>(); public ConfigDepotImpl() { + configCache = Caffeine.newBuilder() + .maximumSize(512) + .expireAfterWrite(CONFIG_CACHE_EXPIRE_SECONDS, TimeUnit.SECONDS) + .build(); ConfigKey.init(this); createEmptyScopeLevelMappings(); } @@ -268,6 +277,48 @@ public class ConfigDepotImpl implements ConfigDepot, ConfigDepotAdmin { return _configDao; } + protected String getConfigStringValueInternal(String cacheKey) { + String[] parts = cacheKey.split("-"); + String key = parts[0]; + ConfigKey.Scope scope = ConfigKey.Scope.Global; + Long scopeId = null; + try { + scope = ConfigKey.Scope.valueOf(parts[1]); + scopeId = Long.valueOf(parts[2]); + } catch (IllegalArgumentException ignored) {} + if (!ConfigKey.Scope.Global.equals(scope) && scopeId != null) { + ScopedConfigStorage scopedConfigStorage = null; + for (ScopedConfigStorage storage : _scopedStorages) { + if (storage.getScope() == scope) { + scopedConfigStorage = storage; + } + } + if (scopedConfigStorage == null) { + throw new CloudRuntimeException("Unable to find config storage for this scope: " + scope + " for " + key); + } + return scopedConfigStorage.getConfigValue(scopeId, key); + } + ConfigurationVO configurationVO = _configDao.findById(key); + if (configurationVO != null) { + return configurationVO.getValue(); + } + return null; + } + + private String getConfigCacheKey(String key, ConfigKey.Scope scope, Long scopeId) { + return String.format("%s-%s-%d", key, scope, (scopeId == null ? 0 : scopeId)); + } + + @Override + public String getConfigStringValue(String key, ConfigKey.Scope scope, Long scopeId) { + return configCache.get(getConfigCacheKey(key, scope, scopeId), this::getConfigStringValueInternal); + } + + @Override + public void invalidateConfigCache(String key, ConfigKey.Scope scope, Long scopeId) { + configCache.invalidate(getConfigCacheKey(key, scope, scopeId)); + } + public ScopedConfigStorage findScopedConfigStorage(ConfigKey config) { for (ScopedConfigStorage storage : _scopedStorages) { if (storage.getScope() == config.scope()) { diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index 8dd6f71af3c..8a7da795345 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -23,12 +23,23 @@ import java.util.HashSet; import java.util.Set; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.junit.Assert; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class ConfigDepotImplTest { + @Mock + ConfigurationDao _configDao; + + @InjectMocks private ConfigDepotImpl configDepotImpl = new ConfigDepotImpl(); @Test @@ -57,4 +68,43 @@ public class ConfigDepotImplTest { Assert.assertFalse(configDepotImpl.isNewConfig(invalidNewConfig)); } + private void runTestGetConfigStringValue(String key, String value) { + ConfigurationVO configurationVO = Mockito.mock(ConfigurationVO.class); + Mockito.when(configurationVO.getValue()).thenReturn(value); + Mockito.when(_configDao.findById(key)).thenReturn(configurationVO); + String result = configDepotImpl.getConfigStringValue(key, ConfigKey.Scope.Global, null); + Assert.assertEquals(value, result); + } + + @Test + public void testGetConfigStringValue() { + runTestGetConfigStringValue("test", "value"); + } + + private void runTestGetConfigStringValueExpiry(long wait, int configDBRetrieval) { + String key = "test1"; + String value = "expiry"; + runTestGetConfigStringValue(key, value); + try { + Thread.sleep(wait); + } catch (InterruptedException ie) { + Assert.fail(ie.getMessage()); + } + String result = configDepotImpl.getConfigStringValue(key, ConfigKey.Scope.Global, null); + Assert.assertEquals(value, result); + Mockito.verify(_configDao, Mockito.times(configDBRetrieval)).findById(key); + + } + + @Test + public void testGetConfigStringValueWithinExpiry() { + runTestGetConfigStringValueExpiry((ConfigDepotImpl.CONFIG_CACHE_EXPIRE_SECONDS * 1000 ) / 4, + 1); + } + + @Test + public void testGetConfigStringValueAfterExpiry() { + runTestGetConfigStringValueExpiry(((ConfigDepotImpl.CONFIG_CACHE_EXPIRE_SECONDS) + 5) * 1000, + 2); + } } diff --git a/pom.xml b/pom.xml index 32811897fa1..2ed3f8301cc 100644 --- a/pom.xml +++ b/pom.xml @@ -187,6 +187,7 @@ 1.4.20 5.3.26 0.5.4 + 3.1.7 @@ -769,6 +770,11 @@ javax.inject 1 + + com.github.ben-manes.caffeine + caffeine + ${cs.caffeine.version} + diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index 47be195ad23..25c866d8609 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -45,17 +45,6 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; - -import com.cloud.dc.VlanDetailsVO; -import com.cloud.dc.dao.VlanDetailsDao; -import com.cloud.hypervisor.HypervisorGuru; -import com.cloud.network.dao.NsxProviderDao; -import com.cloud.network.element.NsxProviderVO; -import com.cloud.utils.crypt.DBEncryptionUtil; -import com.cloud.host.HostTagVO; -import com.cloud.storage.StoragePoolTagVO; -import com.cloud.storage.VolumeApiServiceImpl; -import com.googlecode.ipv6.IPv6Address; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -172,6 +161,7 @@ import com.cloud.dc.Pod; import com.cloud.dc.PodVlanMapVO; import com.cloud.dc.Vlan; import com.cloud.dc.Vlan.VlanType; +import com.cloud.dc.VlanDetailsVO; import com.cloud.dc.VlanVO; import com.cloud.dc.dao.AccountVlanMapDao; import com.cloud.dc.dao.ClusterDao; @@ -185,6 +175,7 @@ import com.cloud.dc.dao.DomainVlanMapDao; import com.cloud.dc.dao.HostPodDao; import com.cloud.dc.dao.PodVlanMapDao; import com.cloud.dc.dao.VlanDao; +import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.dc.dao.VsphereStoragePolicyDao; import com.cloud.deploy.DataCenterDeployment; import com.cloud.deploy.DeploymentClusterPlanner; @@ -203,10 +194,12 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.gpu.GPU; +import com.cloud.host.HostTagVO; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.hypervisor.HypervisorGuru; import com.cloud.hypervisor.kvm.dpdk.DpdkHelper; import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManager; @@ -229,11 +222,13 @@ import com.cloud.network.dao.IPAddressVO; import com.cloud.network.dao.Ipv6GuestPrefixSubnetNetworkMapDao; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.dao.NsxProviderDao; import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkTrafficTypeDao; import com.cloud.network.dao.PhysicalNetworkTrafficTypeVO; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.network.dao.UserIpv6AddressDao; +import com.cloud.network.element.NsxProviderVO; import com.cloud.network.rules.LoadBalancerContainer.Scheme; import com.cloud.network.vpc.VpcManager; import com.cloud.offering.DiskOffering; @@ -261,7 +256,9 @@ import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.storage.Storage.ProvisioningType; import com.cloud.storage.StorageManager; +import com.cloud.storage.StoragePoolTagVO; import com.cloud.storage.Volume; +import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; @@ -281,6 +278,7 @@ import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.db.DB; import com.cloud.utils.db.EntityManager; import com.cloud.utils.db.Filter; @@ -305,6 +303,7 @@ import com.google.common.base.Enums; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import com.googlecode.ipv6.IPv6Address; import com.googlecode.ipv6.IPv6Network; import static com.cloud.configuration.Config.SecStorageAllowedInternalDownloadSites; @@ -705,7 +704,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati value = DBEncryptionUtil.encrypt(value); } - switch (ConfigKey.Scope.valueOf(scope)) { + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + switch (scopeVal) { case Zone: final DataCenterVO zone = _zoneDao.findById(resourceId); if (zone == null) { @@ -796,6 +796,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati throw new InvalidParameterValueException("Scope provided is invalid"); } + _configDepot.invalidateConfigCache(name, scopeVal, resourceId); return valueEncrypted ? DBEncryptionUtil.decrypt(value) : value; } @@ -808,6 +809,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati logger.error("Failed to update configuration option, name: " + name + ", value:" + value); throw new CloudRuntimeException("Failed to update configuration value. Please contact Cloud Support."); } + _configDepot.invalidateConfigCache(name, ConfigKey.Scope.Global, null); PreparedStatement pstmt = null; if (Config.XenServerGuestNetwork.key().equalsIgnoreCase(name)) { @@ -1095,7 +1097,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } String newValue = null; - switch (ConfigKey.Scope.valueOf(scope)) { + ConfigKey.Scope scopeVal = ConfigKey.Scope.valueOf(scope); + switch (scopeVal) { case Zone: final DataCenterVO zone = _zoneDao.findById(id); if (zone == null) { @@ -1180,6 +1183,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati newValue = optionalValue.isPresent() ? optionalValue.get().toString() : defaultValue; } + _configDepot.invalidateConfigCache(name, scopeVal, id); + CallContext.current().setEventDetails(" Name: " + name + " New Value: " + (name.toLowerCase().contains("password") ? "*****" : defaultValue == null ? "" : defaultValue)); return new Pair(_configDao.findByName(name), newValue); } diff --git a/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java b/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java index 0852c20010b..981649758cb 100644 --- a/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java +++ b/server/src/test/java/com/cloud/vm/FirstFitPlannerTest.java @@ -16,6 +16,49 @@ // under the License. package com.cloud.vm; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.ConfigDepot; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.ScopedConfigStorage; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao; +import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; +import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.ComponentScan.Filter; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityManager; import com.cloud.capacity.dao.CapacityDao; @@ -54,48 +97,6 @@ import com.cloud.utils.component.ComponentContext; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; import com.cloud.vm.dao.VMInstanceDao; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.framework.config.ConfigDepot; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.ScopedConfigStorage; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.dao.ConfigurationGroupDao; -import org.apache.cloudstack.framework.config.dao.ConfigurationSubGroupDao; -import org.apache.cloudstack.framework.config.impl.ConfigDepotImpl; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.test.utils.SpringUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mockito; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.ComponentScan.Filter; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.core.type.classreading.MetadataReader; -import org.springframework.core.type.classreading.MetadataReaderFactory; -import org.springframework.core.type.filter.TypeFilter; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - -import javax.inject.Inject; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) @@ -243,9 +244,8 @@ public class FirstFitPlannerTest { } private List initializeForClusterThresholdDisabled() { - ConfigurationVO config = mock(ConfigurationVO.class); - when(config.getValue()).thenReturn(String.valueOf(false)); - when(configDao.findById(DeploymentClusterPlanner.ClusterThresholdEnabled.key())).thenReturn(config); + when(configDepot.getConfigStringValue(DeploymentClusterPlanner.ClusterThresholdEnabled.key(), + ConfigKey.Scope.Global, null)).thenReturn(Boolean.FALSE.toString()); List clustersCrossingThreshold = new ArrayList(); clustersCrossingThreshold.add(3L); diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 1bdcc675346..7ebf9e25734 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -3290,6 +3290,7 @@ "message.set.default.nic": "Please confirm that you would like to make this NIC the default for this Instance.", "message.set.default.nic.manual": "Please manually update the default NIC on the Instance now.", "message.setting.updated": "Setting Updated:", +"message.setting.update.delay": "The new value will take effect within 30 seconds.", "message.setup.physical.network.during.zone.creation": "When adding a zone, you need to set up one or more physical networks. Each physical network can carry one or more types of traffic, with certain restrictions on how they may be combined. Add or remove one or more traffic types onto each physical network.", "message.setup.physical.network.during.zone.creation.basic": "When adding a basic zone, you can set up one physical Network, which corresponds to a NIC on the hypervisor. The Network carries several types of traffic.

You may also add other traffic types onto the physical Network.", "message.shared.network.offering.warning": "Domain admins and regular Users can only create shared Networks from Network offering with the setting specifyvlan=false. Please contact an administrator to create a Network offering if this list is empty.", diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 2f94d6436a4..9bef9349fdb 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -746,7 +746,11 @@ export default { }).then(json => { this.editableValueKey = null this.$store.dispatch('RefreshFeatures') - this.$message.success(`${this.$t('message.setting.updated')} ${record.name}`) + var message = `${this.$t('message.setting.updated')} ${record.name}` + if (record.isdynamic) { + message += `. ${this.$t('message.setting.update.delay')}` + } + this.$message.success(message) if (json.updateconfigurationresponse && json.updateconfigurationresponse.configuration && !json.updateconfigurationresponse.configuration.isdynamic && @@ -767,7 +771,10 @@ export default { api('resetConfiguration', { name: item.name }).then(() => { - const message = `${this.$t('label.setting')} ${item.name} ${this.$t('label.reset.config.value')}` + var message = `${this.$t('label.setting')} ${item.name} ${this.$t('label.reset.config.value')}` + if (item.isdynamic) { + message += `. ${this.$t('message.setting.update.delay')}` + } this.$message.success(message) }).catch(error => { console.error(error) diff --git a/ui/src/components/view/SettingsTab.vue b/ui/src/components/view/SettingsTab.vue index cddbe56b83a..f0812c3a8ad 100644 --- a/ui/src/components/view/SettingsTab.vue +++ b/ui/src/components/view/SettingsTab.vue @@ -174,7 +174,10 @@ export default { name: item.name, value: this.editableValue }).then(() => { - const message = `${this.$t('label.setting')} ${item.name} ${this.$t('label.update.to')} ${this.editableValue}` + var message = `${this.$t('label.setting')} ${item.name} ${this.$t('label.update.to')} ${this.editableValue}` + if (item.isdynamic) { + message += `. ${this.$t('message.setting.update.delay')}` + } this.handleSuccessMessage(item.name, this.$route.meta.name, message) }).catch(error => { console.error(error) @@ -204,7 +207,10 @@ export default { [this.scopeKey]: this.resource.id, name: item.name }).then(() => { - const message = `${this.$t('label.setting')} ${item.name} ${this.$t('label.reset.config.value')}` + var message = `${this.$t('label.setting')} ${item.name} ${this.$t('label.reset.config.value')}` + if (item.isdynamic) { + message += `. ${this.$t('message.setting.update.delay')}` + } this.handleSuccessMessage(item.name, this.$route.meta.name, message) }).catch(error => { console.error(error) diff --git a/ui/src/views/setting/ConfigurationValue.vue b/ui/src/views/setting/ConfigurationValue.vue index 24dd0385741..da6bf399fe2 100644 --- a/ui/src/views/setting/ConfigurationValue.vue +++ b/ui/src/views/setting/ConfigurationValue.vue @@ -258,7 +258,11 @@ export default { this.actualValue = this.editableValue this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') - this.$message.success(`${this.$t('message.setting.updated')} ${configrecord.name}`) + var message = `${this.$t('message.setting.updated')} ${configrecord.name}` + if (configrecord.isdynamic) { + message += `. ${this.$t('message.setting.update.delay')}` + } + this.$message.success(message) if (json.updateconfigurationresponse && json.updateconfigurationresponse.configuration && !json.updateconfigurationresponse.configuration.isdynamic && @@ -295,7 +299,11 @@ export default { } this.$emit('change-config', { value: newValue }) this.$store.dispatch('RefreshFeatures') - this.$message.success(`${this.$t('label.setting')} ${configrecord.name} ${this.$t('label.reset.config.value')}`) + var message = `${this.$t('label.setting')} ${configrecord.name} ${this.$t('label.reset.config.value')}` + if (configrecord.isdynamic) { + message += `. ${this.$t('message.setting.update.delay')}` + } + this.$message.success(message) if (json.resetconfigurationresponse && json.resetconfigurationresponse.configuration && !json.resetconfigurationresponse.configuration.isdynamic && From 7e085d5e1df9fd58305f0a9504c27b35e077e4f0 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 5 Sep 2024 09:36:58 +0530 Subject: [PATCH 29/45] framework/db: use HikariCP as default and improvements (#9518) Per docs, if the mysql connector is JDBC2 compliant then it should use the Connection.isValid API to test a connection. (https://docs.oracle.com/javase/8/docs/api/java/sql/Connection.html#isValid-int-) This would significantly reduce query lags and API throughput, as for every SQL query one or two SELECT 1 are performed everytime a Connection is given to application logic. This should only be accepted when the driver is JDBC4 complaint. As per the docs, the connector-j can use /* ping */ before calling SELECT 1 to have light weight application pings to the server: https://dev.mysql.com/doc/connector-j/en/connector-j-usagenotes-j2ee-concepts-connection-pooling.html Replaces dbcp2 connection pool library with more performant HikariCP. With this unit tests are failing but build is passing. Signed-off-by: Rohit Yadav Signed-off-by: Abhishek Kumar Co-authored-by: Rohit Yadav --- client/conf/db.properties.in | 20 +- developer/pom.xml | 4 + .../com/cloud/upgrade/DatabaseCreator.java | 10 +- engine/storage/snapshot/pom.xml | 5 + framework/db/pom.xml | 8 + .../cloud/utils/db/ConnectionConcierge.java | 2 +- .../com/cloud/utils/db/TransactionLegacy.java | 191 +++++++++++++++--- plugins/network-elements/globodns/pom.xml | 5 + plugins/network-elements/tungsten/pom.xml | 5 + pom.xml | 13 ++ server/pom.xml | 5 + usage/conf/db.properties.in | 4 + 12 files changed, 229 insertions(+), 43 deletions(-) diff --git a/client/conf/db.properties.in b/client/conf/db.properties.in index 8f31aff17e6..0f7d2706a42 100644 --- a/client/conf/db.properties.in +++ b/client/conf/db.properties.in @@ -34,10 +34,14 @@ db.cloud.uri= # CloudStack database tuning parameters +db.cloud.connectionPoolLib=hikaricp db.cloud.maxActive=250 db.cloud.maxIdle=30 -db.cloud.maxWait=10000 -db.cloud.validationQuery=SELECT 1 +db.cloud.maxWait=600000 +db.cloud.minIdleConnections=5 +db.cloud.connectionTimeout=30000 +db.cloud.keepAliveTime=600000 +db.cloud.validationQuery=/* ping */ SELECT 1 db.cloud.testOnBorrow=true db.cloud.testWhileIdle=true db.cloud.timeBetweenEvictionRunsMillis=40000 @@ -70,9 +74,13 @@ db.usage.uri= # usage database tuning parameters +db.usage.connectionPoolLib=hikaricp db.usage.maxActive=100 db.usage.maxIdle=30 -db.usage.maxWait=10000 +db.usage.maxWait=600000 +db.usage.minIdleConnections=5 +db.usage.connectionTimeout=30000 +db.usage.keepAliveTime=600000 db.usage.url.params=serverTimezone=UTC # Simulator database settings @@ -82,9 +90,13 @@ db.simulator.host=@DBHOST@ db.simulator.driver=@DBDRIVER@ db.simulator.port=3306 db.simulator.name=simulator +db.simulator.connectionPoolLib=hikaricp db.simulator.maxActive=250 db.simulator.maxIdle=30 -db.simulator.maxWait=10000 +db.simulator.maxWait=600000 +db.simulator.minIdleConnections=5 +db.simulator.connectionTimeout=30000 +db.simulator.keepAliveTime=600000 db.simulator.autoReconnect=true # Connection URI to the database "simulator". When this property is set, only the following properties will be used along with it: db.simulator.host, db.simulator.port, db.simulator.name, db.simulator.autoReconnect. Other properties will be ignored. diff --git a/developer/pom.xml b/developer/pom.xml index 6cbf483faac..b44446ec0a5 100644 --- a/developer/pom.xml +++ b/developer/pom.xml @@ -42,6 +42,10 @@ org.apache.commons commons-pool2 + + com.zaxxer + HikariCP + org.jasypt jasypt diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java index 154a8d11887..384826227af 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseCreator.java @@ -116,10 +116,6 @@ public class DatabaseCreator { } public static void main(String[] args) { - - ClassPathXmlApplicationContext appContext = new ClassPathXmlApplicationContext(new String[] {"/com/cloud/upgrade/databaseCreatorContext.xml"}); - appContext.getBean(ComponentContext.class); - String dbPropsFile = ""; List sqlFiles = new ArrayList(); List upgradeClasses = new ArrayList(); @@ -166,13 +162,17 @@ public class DatabaseCreator { System.exit(1); } + initDB(dbPropsFile, rootPassword, databases, dryRun); + + ClassPathXmlApplicationContext appContext = new ClassPathXmlApplicationContext(new String[] {"/com/cloud/upgrade/databaseCreatorContext.xml"}); + appContext.getBean(ComponentContext.class); + try { TransactionLegacy.initDataSource(dbPropsFile); } catch (IOException e) { e.printStackTrace(); System.exit(1); } - initDB(dbPropsFile, rootPassword, databases, dryRun); // Process sql files for (String sqlFile : sqlFiles) { diff --git a/engine/storage/snapshot/pom.xml b/engine/storage/snapshot/pom.xml index ac0daeabf76..f29b43d8de0 100644 --- a/engine/storage/snapshot/pom.xml +++ b/engine/storage/snapshot/pom.xml @@ -56,6 +56,11 @@ ${project.version} compile + + mysql + mysql-connector-java + test + diff --git a/framework/db/pom.xml b/framework/db/pom.xml index d4d3b6b8772..586d72f34f3 100644 --- a/framework/db/pom.xml +++ b/framework/db/pom.xml @@ -40,6 +40,10 @@ org.apache.commons commons-dbcp2 + + com.zaxxer + HikariCP + commons-io commons-io @@ -48,6 +52,10 @@ org.apache.commons commons-pool2 + + mysql + mysql-connector-java + org.apache.cloudstack cloud-utils diff --git a/framework/db/src/main/java/com/cloud/utils/db/ConnectionConcierge.java b/framework/db/src/main/java/com/cloud/utils/db/ConnectionConcierge.java index 7cf34e6955c..c7c869ba920 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/ConnectionConcierge.java +++ b/framework/db/src/main/java/com/cloud/utils/db/ConnectionConcierge.java @@ -145,7 +145,7 @@ public class ConnectionConcierge { protected String testValidity(String name, Connection conn) { if (conn != null) { synchronized (conn) { - try (PreparedStatement pstmt = conn.prepareStatement("SELECT 1");) { + try (PreparedStatement pstmt = conn.prepareStatement("/* ping */ SELECT 1");) { pstmt.executeQuery(); } catch (Throwable th) { logger.error("Unable to keep the db connection for " + name, th); diff --git a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java index 00fa8e4c0d5..88af397c06a 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java +++ b/framework/db/src/main/java/com/cloud/utils/db/TransactionLegacy.java @@ -38,17 +38,20 @@ import org.apache.commons.dbcp2.DriverManagerConnectionFactory; import org.apache.commons.dbcp2.PoolableConnection; import org.apache.commons.dbcp2.PoolableConnectionFactory; import org.apache.commons.dbcp2.PoolingDataSource; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.pool2.ObjectPool; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.utils.Pair; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.mgmt.JmxUtil; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; /** * Transaction abstracts away the Connection object in JDBC. It allows the @@ -95,6 +98,8 @@ public class TransactionLegacy implements Closeable { } } + private static final String CONNECTION_POOL_LIB_DBCP = "dbcp"; + private final LinkedList _stack; private long _id; @@ -1020,6 +1025,21 @@ public class TransactionLegacy implements Closeable { } } + private static T parseNumber(String value, Class type) { + if (value == null) { + return null; + } + try { + if (type.equals(Long.class)) { + return type.cast(Long.parseLong(value)); + } else { + return type.cast(Integer.parseInt(value)); + } + } catch (NumberFormatException ignored) { + return null; + } + } + @SuppressWarnings({"rawtypes", "unchecked"}) public static void initDataSource(Properties dbProps) { try { @@ -1030,9 +1050,12 @@ public class TransactionLegacy implements Closeable { LOGGER.info("Is Data Base High Availiability enabled? Ans : " + s_dbHAEnabled); String loadBalanceStrategy = dbProps.getProperty("db.ha.loadBalanceStrategy"); // FIXME: If params are missing...default them???? - final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive")); - final int cloudMaxIdle = Integer.parseInt(dbProps.getProperty("db.cloud.maxIdle")); - final long cloudMaxWait = Long.parseLong(dbProps.getProperty("db.cloud.maxWait")); + final Integer cloudMaxActive = parseNumber(dbProps.getProperty("db.cloud.maxActive"), Integer.class); + final Integer cloudMaxIdle = parseNumber(dbProps.getProperty("db.cloud.maxIdle"), Integer.class); + final Long cloudMaxWait = parseNumber(dbProps.getProperty("db.cloud.maxWait"), Long.class); + final Integer cloudMinIdleConnections = parseNumber(dbProps.getProperty("db.cloud.minIdleConnections"), Integer.class); + final Long cloudConnectionTimeout = parseNumber(dbProps.getProperty("db.cloud.connectionTimeout"), Long.class); + final Long cloudKeepAliveTimeout = parseNumber(dbProps.getProperty("db.cloud.keepAliveTime"), Long.class); final String cloudUsername = dbProps.getProperty("db.cloud.username"); final String cloudPassword = dbProps.getProperty("db.cloud.password"); final String cloudValidationQuery = dbProps.getProperty("db.cloud.validationQuery"); @@ -1071,14 +1094,19 @@ public class TransactionLegacy implements Closeable { DriverLoader.loadDriver(cloudUriAndDriver.second()); // Default Data Source for CloudStack - s_ds = createDataSource(cloudUriAndDriver.first(), cloudUsername, cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait, - cloudTimeBtwEvictionRunsMillis, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle, cloudTestOnBorrow, - cloudValidationQuery, isolationLevel); + s_ds = createDataSource(dbProps.getProperty("db.cloud.connectionPoolLib"), cloudUriAndDriver.first(), + cloudUsername, cloudPassword, cloudMaxActive, cloudMaxIdle, cloudMaxWait, + cloudTimeBtwEvictionRunsMillis, cloudMinEvcitableIdleTimeMillis, cloudTestWhileIdle, + cloudTestOnBorrow, cloudValidationQuery, cloudMinIdleConnections, cloudConnectionTimeout, + cloudKeepAliveTimeout, isolationLevel, "cloud"); // Configure the usage db - final int usageMaxActive = Integer.parseInt(dbProps.getProperty("db.usage.maxActive")); - final int usageMaxIdle = Integer.parseInt(dbProps.getProperty("db.usage.maxIdle")); - final long usageMaxWait = Long.parseLong(dbProps.getProperty("db.usage.maxWait")); + final Integer usageMaxActive = parseNumber(dbProps.getProperty("db.usage.maxActive"), Integer.class); + final Integer usageMaxIdle = parseNumber(dbProps.getProperty("db.usage.maxIdle"), Integer.class); + final Long usageMaxWait = parseNumber(dbProps.getProperty("db.usage.maxWait"), Long.class); + final Integer usageMinIdleConnections = parseNumber(dbProps.getProperty("db.usage.minIdleConnections"), Integer.class); + final Long usageConnectionTimeout = parseNumber(dbProps.getProperty("db.usage.connectionTimeout"), Long.class); + final Long usageKeepAliveTimeout = parseNumber(dbProps.getProperty("db.usage.keepAliveTime"), Long.class); final String usageUsername = dbProps.getProperty("db.usage.username"); final String usagePassword = dbProps.getProperty("db.usage.password"); @@ -1087,15 +1115,19 @@ public class TransactionLegacy implements Closeable { DriverLoader.loadDriver(usageUriAndDriver.second()); // Data Source for usage server - s_usageDS = createDataSource(usageUriAndDriver.first(), usageUsername, usagePassword, - usageMaxActive, usageMaxIdle, usageMaxWait, null, null, null, null, - null, isolationLevel); + s_usageDS = createDataSource(dbProps.getProperty("db.usage.connectionPoolLib"), usageUriAndDriver.first(), + usageUsername, usagePassword, usageMaxActive, usageMaxIdle, usageMaxWait, null, + null, null, null, null, + usageMinIdleConnections, usageConnectionTimeout, usageKeepAliveTimeout, isolationLevel, "usage"); try { // Configure the simulator db - final int simulatorMaxActive = Integer.parseInt(dbProps.getProperty("db.simulator.maxActive")); - final int simulatorMaxIdle = Integer.parseInt(dbProps.getProperty("db.simulator.maxIdle")); - final long simulatorMaxWait = Long.parseLong(dbProps.getProperty("db.simulator.maxWait")); + final Integer simulatorMaxActive = parseNumber(dbProps.getProperty("db.simulator.maxActive"), Integer.class); + final Integer simulatorMaxIdle = parseNumber(dbProps.getProperty("db.simulator.maxIdle"), Integer.class); + final Long simulatorMaxWait = parseNumber(dbProps.getProperty("db.simulator.maxWait"), Long.class); + final Integer simulatorMinIdleConnections = parseNumber(dbProps.getProperty("db.simulator.minIdleConnections"), Integer.class); + final Long simulatorConnectionTimeout = parseNumber(dbProps.getProperty("db.simulator.connectionTimeout"), Long.class); + final Long simulatorKeepAliveTimeout = parseNumber(dbProps.getProperty("db.simulator.keepAliveTime"), Long.class); final String simulatorUsername = dbProps.getProperty("db.simulator.username"); final String simulatorPassword = dbProps.getProperty("db.simulator.password"); @@ -1122,15 +1154,18 @@ public class TransactionLegacy implements Closeable { DriverLoader.loadDriver(simulatorDriver); - s_simulatorDS = createDataSource(simulatorConnectionUri, simulatorUsername, simulatorPassword, - simulatorMaxActive, simulatorMaxIdle, simulatorMaxWait, null, null, null, null, cloudValidationQuery, isolationLevel); + s_simulatorDS = createDataSource(dbProps.getProperty("db.simulator.connectionPoolLib"), + simulatorConnectionUri, simulatorUsername, simulatorPassword, simulatorMaxActive, + simulatorMaxIdle, simulatorMaxWait, null, null, null, null, + cloudValidationQuery, simulatorMinIdleConnections, simulatorConnectionTimeout, + simulatorKeepAliveTimeout, isolationLevel, "simulator"); } catch (Exception e) { LOGGER.debug("Simulator DB properties are not available. Not initializing simulator DS"); } } catch (final Exception e) { - s_ds = getDefaultDataSource("cloud"); - s_usageDS = getDefaultDataSource("cloud_usage"); - s_simulatorDS = getDefaultDataSource("cloud_simulator"); + s_ds = getDefaultDataSource(dbProps.getProperty("db.cloud.connectionPoolLib"), "cloud"); + s_usageDS = getDefaultDataSource(dbProps.getProperty("db.usage.connectionPoolLib"), "cloud_usage"); + s_simulatorDS = getDefaultDataSource(dbProps.getProperty("db.simulator.connectionPoolLib"), "simulator"); LOGGER.warn( "Unable to load db configuration, using defaults with 5 connections. Falling back on assumed datasource on localhost:3306 using username:password=cloud:cloud. Please check your configuration", e); @@ -1222,11 +1257,71 @@ public class TransactionLegacy implements Closeable { /** * Creates a data source */ - private static DataSource createDataSource(String uri, String username, String password, + private static DataSource createDataSource(String connectionPoolLib, String uri, String username, String password, + Integer maxActive, Integer maxIdle, Long maxWait, Long timeBtwnEvictionRuns, Long minEvictableIdleTime, + Boolean testWhileIdle, Boolean testOnBorrow, String validationQuery, Integer minIdleConnections, + Long connectionTimeout, Long keepAliveTime, Integer isolationLevel, String dsName) { + LOGGER.debug("Creating datasource for database: {} with connection pool lib: {}", dsName, + connectionPoolLib); + if (CONNECTION_POOL_LIB_DBCP.equals(connectionPoolLib)) { + return createDbcpDataSource(uri, username, password, maxActive, maxIdle, maxWait, timeBtwnEvictionRuns, + minEvictableIdleTime, testWhileIdle, testOnBorrow, validationQuery, isolationLevel); + } + return createHikaricpDataSource(uri, username, password, maxActive, maxIdle, maxWait, minIdleConnections, + connectionTimeout, keepAliveTime, isolationLevel, dsName); + } + + private static DataSource createHikaricpDataSource(String uri, String username, String password, Integer maxActive, Integer maxIdle, Long maxWait, - Long timeBtwnEvictionRuns, Long minEvictableIdleTime, - Boolean testWhileIdle, Boolean testOnBorrow, - String validationQuery, Integer isolationLevel) { + Integer minIdleConnections, Long connectionTimeout, Long keepAliveTime, + Integer isolationLevel, String dsName) { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl(uri); + config.setUsername(username); + config.setPassword(password); + + config.setPoolName(dsName); + + // Connection pool properties + config.setMaximumPoolSize(ObjectUtils.defaultIfNull(maxActive, 250)); + config.setIdleTimeout(ObjectUtils.defaultIfNull(maxIdle, 30) * 1000); + config.setMaxLifetime(ObjectUtils.defaultIfNull(maxWait, 600000L)); + config.setMinimumIdle(ObjectUtils.defaultIfNull(minIdleConnections, 5)); + config.setConnectionTimeout(ObjectUtils.defaultIfNull(connectionTimeout, 30000L)); + config.setKeepaliveTime(ObjectUtils.defaultIfNull(keepAliveTime, 600000L)); + + String isolationLevelString = "TRANSACTION_READ_COMMITTED"; + if (isolationLevel == Connection.TRANSACTION_SERIALIZABLE) { + isolationLevelString = "TRANSACTION_SERIALIZABLE"; + } else if (isolationLevel == Connection.TRANSACTION_READ_UNCOMMITTED) { + isolationLevelString = "TRANSACTION_READ_UNCOMMITTED"; + } else if (isolationLevel == Connection.TRANSACTION_REPEATABLE_READ) { + isolationLevelString = "TRANSACTION_REPEATABLE_READ"; + } + config.setTransactionIsolation(isolationLevelString); + + // Standard datasource config for MySQL + config.addDataSourceProperty("cachePrepStmts", "true"); + config.addDataSourceProperty("prepStmtCacheSize", "250"); + config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048"); + // Additional config for MySQL + config.addDataSourceProperty("useServerPrepStmts", "true"); + config.addDataSourceProperty("useLocalSessionState", "true"); + config.addDataSourceProperty("rewriteBatchedStatements", "true"); + config.addDataSourceProperty("cacheResultSetMetadata", "true"); + config.addDataSourceProperty("cacheServerConfiguration", "true"); + config.addDataSourceProperty("elideSetAutoCommits", "true"); + config.addDataSourceProperty("maintainTimeStats", "false"); + + HikariDataSource dataSource = new HikariDataSource(config); + return dataSource; + } + + private static DataSource createDbcpDataSource(String uri, String username, String password, + Integer maxActive, Integer maxIdle, Long maxWait, + Long timeBtwnEvictionRuns, Long minEvictableIdleTime, + Boolean testWhileIdle, Boolean testOnBorrow, + String validationQuery, Integer isolationLevel) { ConnectionFactory connectionFactory = new DriverManagerConnectionFactory(uri, username, password); PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(connectionFactory, null); GenericObjectPoolConfig config = createPoolConfig(maxActive, maxIdle, maxWait, timeBtwnEvictionRuns, minEvictableIdleTime, testWhileIdle, testOnBorrow); @@ -1267,6 +1362,44 @@ public class TransactionLegacy implements Closeable { return config; } + private static DataSource getDefaultDataSource(final String connectionPoolLib, final String database) { + LOGGER.debug("Creating default datasource for database: {} with connection pool lib: {}", + database, connectionPoolLib); + if (CONNECTION_POOL_LIB_DBCP.equalsIgnoreCase(connectionPoolLib)) { + return getDefaultDbcpDataSource(database); + } + return getDefaultHikaricpDataSource(database); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static DataSource getDefaultHikaricpDataSource(final String database) { + HikariConfig config = new HikariConfig(); + config.setJdbcUrl("jdbc:mysql://localhost:3306/" + database + "?" + CONNECTION_PARAMS); + config.setUsername("cloud"); + config.setPassword("cloud"); + config.setPoolName(database); + config.setDriverClassName("com.mysql.cj.jdbc.Driver"); + config.setMaximumPoolSize(250); + config.setConnectionTimeout(1000); + config.setIdleTimeout(1000); + config.setKeepaliveTime(1000); + config.setMaxLifetime(1000); + config.setTransactionIsolation("TRANSACTION_READ_COMMITTED"); + config.setInitializationFailTimeout(-1L); + config.addDataSourceProperty("cachePrepStmts", "true"); + config.addDataSourceProperty("prepStmtCacheSize", "250"); + config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048"); + return new HikariDataSource(config); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static DataSource getDefaultDbcpDataSource(final String database) { + final ConnectionFactory connectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://localhost:3306/" + database + "?" + CONNECTION_PARAMS, "cloud", "cloud"); + final PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(connectionFactory, null); + final GenericObjectPool connectionPool = new GenericObjectPool(poolableConnectionFactory); + return new PoolingDataSource(connectionPool); + } + private static String getDBHAParams(String dbName, Properties dbProps) { StringBuilder sb = new StringBuilder(); sb.append("failOverReadOnly=" + dbProps.getProperty("db." + dbName + ".failOverReadOnly")); @@ -1278,14 +1411,6 @@ public class TransactionLegacy implements Closeable { return sb.toString(); } - @SuppressWarnings({"unchecked", "rawtypes"}) - private static DataSource getDefaultDataSource(final String database) { - final ConnectionFactory connectionFactory = new DriverManagerConnectionFactory("jdbc:mysql://localhost:3306/" + database + "?" + CONNECTION_PARAMS, "cloud", "cloud"); - final PoolableConnectionFactory poolableConnectionFactory = new PoolableConnectionFactory(connectionFactory, null); - final GenericObjectPool connectionPool = new GenericObjectPool(poolableConnectionFactory); - return new PoolingDataSource(connectionPool); - } - /** * Used for unit testing primarily * diff --git a/plugins/network-elements/globodns/pom.xml b/plugins/network-elements/globodns/pom.xml index e27a4a54d16..c0200921f20 100644 --- a/plugins/network-elements/globodns/pom.xml +++ b/plugins/network-elements/globodns/pom.xml @@ -32,5 +32,10 @@ com.globo.globodns globodns-client + + mysql + mysql-connector-java + test + diff --git a/plugins/network-elements/tungsten/pom.xml b/plugins/network-elements/tungsten/pom.xml index b0a46b1331a..1345268d313 100644 --- a/plugins/network-elements/tungsten/pom.xml +++ b/plugins/network-elements/tungsten/pom.xml @@ -47,5 +47,10 @@ ch.qos.reload4j reload4j + + mysql + mysql-connector-java + test + diff --git a/pom.xml b/pom.xml index 2ed3f8301cc..1acea98096f 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,7 @@ 1.10 1.3.3 2.9.0 + 5.1.0 0.5 2.6 2.9.0 @@ -364,6 +365,7 @@ commons-daemon ${cs.daemon.version} + org.apache.commons commons-dbcp2 @@ -375,6 +377,11 @@ + + com.zaxxer + HikariCP + ${cs.hikaricp.version} + commons-discovery commons-discovery @@ -455,6 +462,12 @@ reload4j ${cs.reload4j.version} + + mysql + mysql-connector-java + ${cs.mysql.version} + test + log4j apache-log4j-extras diff --git a/server/pom.xml b/server/pom.xml index 269583c381a..ec157b00e30 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -164,6 +164,11 @@ cloud-framework-agent-lb ${project.version} + + org.apache.cloudstack + cloud-framework-db + ${project.version} + org.apache.cloudstack cloud-engine-storage-configdrive diff --git a/usage/conf/db.properties.in b/usage/conf/db.properties.in index 35a40def7b1..c597bb9fad3 100644 --- a/usage/conf/db.properties.in +++ b/usage/conf/db.properties.in @@ -24,7 +24,11 @@ db.usage.port=3306 db.usage.name=cloud_usage # usage database tuning parameters +db.usage.connectionPoolLib=hikaricp db.usage.maxActive=100 db.usage.maxIdle=30 db.usage.maxWait=10000 +db.usage.minIdleConnections=5 +db.usage.connectionTimeout=30000 +db.usage.keepAliveTime=600000 db.usage.autoReconnect=true From 36d37f70a8246b742706db0139059337085a56f2 Mon Sep 17 00:00:00 2001 From: Nicolas Vazquez Date: Thu, 5 Sep 2024 01:09:07 -0300 Subject: [PATCH 30/45] Display associated resource name on storage pools objects (#9449) --- .../browser/DataStoreObjectResponse.java | 36 +++++++++++++++++++ .../storage/browser/StorageBrowserImpl.java | 14 +++++--- ui/src/views/infra/StorageBrowser.vue | 8 ++--- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java b/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java index cac5cc91b03..c281fa115fd 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java @@ -41,6 +41,10 @@ public class DataStoreObjectResponse extends BaseResponse { @Param(description = "Template ID associated with the data store object.") private String templateId; + @SerializedName(ApiConstants.TEMPLATE_NAME) + @Param(description = "Template Name associated with the data store object.") + private String templateName; + @SerializedName(ApiConstants.FORMAT) @Param(description = "Format of template associated with the data store object.") private String format; @@ -49,10 +53,18 @@ public class DataStoreObjectResponse extends BaseResponse { @Param(description = "Snapshot ID associated with the data store object.") private String snapshotId; + @SerializedName("snapshotname") + @Param(description = "Snapshot Name associated with the data store object.") + private String snapshotName; + @SerializedName(ApiConstants.VOLUME_ID) @Param(description = "Volume ID associated with the data store object.") private String volumeId; + @SerializedName(ApiConstants.VOLUME_NAME) + @Param(description = "Volume Name associated with the data store object.") + private String volumeName; + @SerializedName(ApiConstants.LAST_UPDATED) @Param(description = "Last modified date of the file/directory.") private Date lastUpdated; @@ -86,6 +98,18 @@ public class DataStoreObjectResponse extends BaseResponse { this.volumeId = volumeId; } + public void setTemplateName(String templateName) { + this.templateName = templateName; + } + + public void setSnapshotName(String snapshotName) { + this.snapshotName = snapshotName; + } + + public void setVolumeName(String volumeName) { + this.volumeName = volumeName; + } + public String getName() { return name; } @@ -117,4 +141,16 @@ public class DataStoreObjectResponse extends BaseResponse { public Date getLastUpdated() { return lastUpdated; } + + public String getTemplateName() { + return templateName; + } + + public String getSnapshotName() { + return snapshotName; + } + + public String getVolumeName() { + return volumeName; + } } diff --git a/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java b/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java index 8828ac486f5..daa20464f3b 100644 --- a/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java +++ b/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java @@ -233,14 +233,20 @@ public class StorageBrowserImpl extends MutualExclusiveIdsManagerBase implements new Date(answer.getLastModified().get(i))); String filePath = paths.get(i); if (pathTemplateMap.get(filePath) != null) { - response.setTemplateId(pathTemplateMap.get(filePath).getUuid()); - response.setFormat(pathTemplateMap.get(filePath).getFormat().toString()); + VMTemplateVO vmTemplateVO = pathTemplateMap.get(filePath); + response.setTemplateId(vmTemplateVO.getUuid()); + response.setFormat(vmTemplateVO.getFormat().toString()); + response.setTemplateName(vmTemplateVO.getName()); } if (pathSnapshotMap.get(filePath) != null) { - response.setSnapshotId(pathSnapshotMap.get(filePath).getUuid()); + SnapshotVO snapshotVO = pathSnapshotMap.get(filePath); + response.setSnapshotId(snapshotVO.getUuid()); + response.setSnapshotName(snapshotVO.getName()); } if (pathVolumeMap.get(filePath) != null) { - response.setVolumeId(pathVolumeMap.get(filePath).getUuid()); + VolumeVO volumeVO = pathVolumeMap.get(filePath); + response.setVolumeId(volumeVO.getUuid()); + response.setVolumeName(volumeVO.getName()); } responses.add(response); } diff --git a/ui/src/views/infra/StorageBrowser.vue b/ui/src/views/infra/StorageBrowser.vue index ce3464ba0c8..f547737b281 100644 --- a/ui/src/views/infra/StorageBrowser.vue +++ b/ui/src/views/infra/StorageBrowser.vue @@ -127,20 +127,20 @@ + + diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index 6d694f4f6d6..e8b0db974e6 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -181,7 +181,7 @@ -