Merge branch '4.19'

api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupsCmd.java

@@ -116,6 +116,7 @@ public class ListBackupsCmd extends BaseListProjectAndAccountResourcesCmd {
             Pair<List<Backup>, Integer> result = backupManager.listBackups(this);
             setupResponseBackupList(result.first(), result.second());
         } catch (Exception e) {
+            logger.debug("Exception while listing backups", e);
             throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage());
         }
     }

api/src/main/java/org/apache/cloudstack/backup/Backup.java

@@ -134,6 +134,7 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity {
     }
 
     long getVmId();
+    long getBackupOfferingId();
     String getExternalId();
     String getType();
     Date getDate();

engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java

@@ -29,6 +29,7 @@ import com.cloud.utils.db.QueryBuilder;
 import com.cloud.utils.db.SearchCriteria;
 import com.cloud.utils.db.Transaction;
 import com.cloud.utils.db.TransactionCallback;
+import com.cloud.utils.db.TransactionCallbackNoReturn;
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.TransactionStatus;
 import com.cloud.utils.exception.CloudRuntimeException;
@@ -538,21 +539,25 @@ public class UsageDaoImpl extends GenericDaoBase<UsageVO, Long> implements Usage
 
     @Override
     public void removeOldUsageRecords(int days) {
-        String sql = DELETE_ALL_BY_INTERVAL;
+        Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallbackNoReturn() {
-        TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB);
+            @Override
-        PreparedStatement pstmt = null;
+            public void doInTransactionWithoutResult(TransactionStatus status) {
-        try {
+                TransactionLegacy txn = TransactionLegacy.currentTxn();
-            txn.start();
+                PreparedStatement pstmt = null;
-            pstmt = txn.prepareAutoCloseStatement(sql);
+                try {
-            pstmt.setLong(1, days);
+                    txn.start();
-            pstmt.executeUpdate();
+                    pstmt = txn.prepareAutoCloseStatement(DELETE_ALL_BY_INTERVAL);
-            txn.commit();
+                    pstmt.setLong(1, days);
-        } catch (Exception ex) {
+                    pstmt.executeUpdate();
-            txn.rollback();
+                    txn.commit();
-            logger.error("error removing old cloud_usage records for interval: " + days);
+                } catch (Exception ex) {
-        } finally {
+                    txn.rollback();
-            txn.close();
+                    logger.error("error removing old cloud_usage records for interval: " + days);
-        }
+                } finally {
+                    txn.close();
+                }
+            }
+        });
     }
 
     public UsageVO persistUsage(final UsageVO usage) {

engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java

@@ -856,6 +856,7 @@ public class VMInstanceDaoImpl extends GenericDaoBase<VMInstanceVO, Long> implem
 
             SearchBuilder<NicVO> nicSearch = nicDao.createSearchBuilder();
             nicSearch.and("networkId", nicSearch.entity().getNetworkId(), SearchCriteria.Op.EQ);
+            nicSearch.and("removed", nicSearch.entity().getRemoved(), SearchCriteria.Op.NULL);
 
             NetworkTypeSearch = createSearchBuilder();
             NetworkTypeSearch.and("types", NetworkTypeSearch.entity().getType(), SearchCriteria.Op.IN);

engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java

@@ -158,6 +158,7 @@ public class BackupVO implements Backup {
         this.status = status;
     }
 
+    @Override
     public long getBackupOfferingId() {
         return backupOfferingId;
     }

engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupDaoImpl.java

@@ -145,7 +145,11 @@ public class BackupDaoImpl extends GenericDaoBase<BackupVO, Long> implements Bac
         AccountVO account = accountDao.findByIdIncludingRemoved(vm.getAccountId());
         DomainVO domain = domainDao.findByIdIncludingRemoved(vm.getDomainId());
         DataCenterVO zone = dataCenterDao.findByIdIncludingRemoved(vm.getDataCenterId());
-        BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(vm.getBackupOfferingId());
+        Long offeringId = vm.getBackupOfferingId();
+        if (offeringId == null) {
+            offeringId = backup.getBackupOfferingId();
+        }
+        BackupOffering offering = backupOfferingDao.findByIdIncludingRemoved(offeringId);
 
         BackupResponse response = new BackupResponse();
         response.setId(backup.getUuid());

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtOvsFetchInterfaceCommandWrapperTest.java

@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.UUID;
 
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.Spy;
@@ -49,8 +50,7 @@ public class LibvirtOvsFetchInterfaceCommandWrapperTest {
             while(interfaces.hasMoreElements()) {
                 NetworkInterface networkInterface = interfaces.nextElement();
                 if (networkInterface.getInetAddresses().hasMoreElements() &&
-                        (networkInterface.getName().startsWith("eth") ||
+                        networkInterface.getName().matches("^(eth|wl|en).*")) {
-                                networkInterface.getName().startsWith("wl"))) {
                     interfaceName = networkInterface.getName();
                     Enumeration<InetAddress> addresses = networkInterface.getInetAddresses();
                     while(addresses.hasMoreElements()) {
@@ -60,9 +60,13 @@ public class LibvirtOvsFetchInterfaceCommandWrapperTest {
                             break;
                         };
                     }
+                    if (StringUtils.isNotBlank(interfaceName) && StringUtils.isNotBlank(ipAddress)) {
+                        break;
+                    }
                 }
             }
         } catch (SocketException ignored) {}
+        Assume.assumeTrue(StringUtils.isNotBlank(interfaceName));
         Ternary<String, String, String> result = null;
         try {
             result = wrapper.getInterfaceDetails(interfaceName);

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmd.java

@@ -226,6 +226,7 @@ public class SAML2LoginAPIAuthenticatorCmd extends BaseCmd implements APIAuthent
                             "Received SAML response for a SSO request that we may not have made or has expired, please try logging in again",
                             params, responseType));
                 }
+                samlAuthManager.purgeToken(token);
 
                 // Set IdpId for this session
                 session.setAttribute(SAMLPluginConstants.SAML_IDPID, issuer.getValue());

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManager.java

@@ -71,16 +71,17 @@ public interface SAML2AuthManager extends PluggableAPIAuthenticator, PluggableSe
             "SAML2 IDP Metadata refresh interval in seconds, minimum value is set to 300", true);
 
     ConfigKey<Boolean> SAMLCheckSignature = new ConfigKey<Boolean>("Advanced", Boolean.class, "saml2.check.signature", "true",
-            "Whether SAML2 signature must be checked, when enforced and when the SAML response does not have a signature would lead to login exception", true);
+            "When enabled (default and recommended), SAML2 signature checks are enforced and lack of signature in the SAML SSO response will cause login exception. Disabling this is not advisable but provided for backward compatibility for users who are able to accept the risks.", false);
 
-    public SAMLProviderMetadata getSPMetadata();
+    SAMLProviderMetadata getSPMetadata();
-    public SAMLProviderMetadata getIdPMetadata(String entityId);
+    SAMLProviderMetadata getIdPMetadata(String entityId);
-    public Collection<SAMLProviderMetadata> getAllIdPMetadata();
+    Collection<SAMLProviderMetadata> getAllIdPMetadata();
 
-    public boolean isUserAuthorized(Long userId, String entityId);
+    boolean isUserAuthorized(Long userId, String entityId);
-    public boolean authorizeUser(Long userId, String entityId, boolean enable);
+    boolean authorizeUser(Long userId, String entityId, boolean enable);
 
-    public void saveToken(String authnId, String domain, String entity);
+    void saveToken(String authnId, String domain, String entity);
-    public SAMLTokenVO getToken(String authnId);
+    SAMLTokenVO getToken(String authnId);
-    public void expireTokens();
+    void purgeToken(SAMLTokenVO token);
+    void expireTokens();
 }

plugins/user-authenticators/saml2/src/main/java/org/apache/cloudstack/saml/SAML2AuthManagerImpl.java

@@ -485,6 +485,13 @@ public class SAML2AuthManagerImpl extends AdapterBase implements SAML2AuthManage
         return _samlTokenDao.findByUuid(authnId);
     }
 
+    @Override
+    public void purgeToken(SAMLTokenVO token) {
+        if (token != null) {
+            _samlTokenDao.remove(token.getId());
+        }
+    }
+
     @Override
     public void expireTokens() {
         _samlTokenDao.expireTokens();

plugins/user-authenticators/saml2/src/test/java/org/apache/cloudstack/api/command/SAML2LoginAPIAuthenticatorCmdTest.java

@@ -279,7 +279,7 @@ public class SAML2LoginAPIAuthenticatorCmdTest {
 
     @Test
     public void testFailOnSAMLSignatureCheckWhenFalse() throws NoSuchFieldException, IllegalAccessException {
-        overrideDefaultConfigValue(SAML2AuthManager.SAMLCheckSignature, "_defaultValue", "false");
+        overrideDefaultConfigValue(SAML2AuthManager.SAMLCheckSignature, "_value", false);
         SAML2LoginAPIAuthenticatorCmd cmd = new SAML2LoginAPIAuthenticatorCmd();
         try {
             cmd.checkAndFailOnMissingSAMLSignature(null);
@@ -290,7 +290,7 @@ public class SAML2LoginAPIAuthenticatorCmdTest {
 
     @Test(expected = ServerApiException.class)
     public void testFailOnSAMLSignatureCheckWhenTrue() throws NoSuchFieldException, IllegalAccessException {
-        overrideDefaultConfigValue(SAML2AuthManager.SAMLCheckSignature, "_defaultValue", "true");
+        overrideDefaultConfigValue(SAML2AuthManager.SAMLCheckSignature, "_value", true);
         SAML2LoginAPIAuthenticatorCmd cmd = new SAML2LoginAPIAuthenticatorCmd();
         cmd.checkAndFailOnMissingSAMLSignature(null);
     }

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -2791,14 +2791,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
 
     @Override
     public boolean canHostPrepareStoragePoolAccess(Host host, StoragePool pool) {
-        if (host == null || pool == null) {
+        if (host == null || pool == null || !pool.isManaged()) {
             return false;
         }
 
-        if (!pool.isManaged()) {
-            return true;
-        }
-
         DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName());
         DataStoreDriver storeDriver = storeProvider.getDataStoreDriver();
         return storeDriver instanceof PrimaryDataStoreDriver && ((PrimaryDataStoreDriver)storeDriver).canHostPrepareStoragePoolAccess(host, pool);

server/src/test/java/com/cloud/event/ActionEventInterceptorTest.java

@@ -140,6 +140,7 @@ public class ActionEventInterceptorTest {
         }
 
         utils.init();
+        CallContext.register(user, account);
     }
 
     /**
@@ -152,6 +153,7 @@ public class ActionEventInterceptorTest {
         Mockito.when(configDao.getValue(Config.PublishActionEvent.key())).thenReturn("true");
         componentContextMocked = Mockito.mockStatic(ComponentContext.class);
         Mockito.when(ComponentContext.getComponent(EventBus.class)).thenReturn(eventBus);
+        persistedEvents = new ArrayList<>();
 
         //Needed for persist to actually set an ID that can be returned from the ActionEventUtils
         //methods.
@@ -231,11 +233,11 @@ public class ActionEventInterceptorTest {
         Object event = actionEventInterceptor.interceptStart(m, tester);
         Assert.assertNull(event);
 
-        Assert.assertEquals(persistedEvents.size(), 1);
+        Assert.assertEquals(1, persistedEvents.size());
         EventVO eventVO = persistedEvents.get(0);
-        Assert.assertEquals(eventVO.getType(), EventTypes.EVENT_VM_START);
+        Assert.assertEquals(EventTypes.EVENT_VM_START, eventVO.getType());
-        Assert.assertEquals(eventVO.getDescription(), "Starting VM");
+        Assert.assertEquals(eventDescription, eventVO.getDescription());
-        Assert.assertEquals(eventVO.getState(), com.cloud.event.Event.State.Started);
+        Assert.assertEquals(com.cloud.event.Event.State.Started, eventVO.getState());
     }
 
     @Test
@@ -244,12 +246,12 @@ public class ActionEventInterceptorTest {
         Method m = tester.getClass().getMethod("testMethod");
         actionEventInterceptor.interceptComplete(m, tester, null);
 
-        Assert.assertEquals(persistedEvents.size(), 1);
+        Assert.assertEquals(1, persistedEvents.size());
         EventVO eventVO = persistedEvents.get(0);
-        Assert.assertEquals(eventVO.getType(), eventType);
+        Assert.assertEquals(eventType, eventVO.getType());
         Assert.assertTrue(eventVO.getDescription().endsWith(eventDescription));
-        Assert.assertEquals(eventVO.getLevel(), EventVO.LEVEL_INFO);
+        Assert.assertEquals(EventVO.LEVEL_INFO, eventVO.getLevel());
-        Assert.assertEquals(eventVO.getState(), com.cloud.event.Event.State.Completed);
+        Assert.assertEquals(com.cloud.event.Event.State.Completed, eventVO.getState());
     }
 
     @Test
@@ -258,12 +260,12 @@ public class ActionEventInterceptorTest {
         Method m = tester.getClass().getMethod("testMethod");
         actionEventInterceptor.interceptException(m, tester, null);
 
-        Assert.assertEquals(persistedEvents.size(), 1);
+        Assert.assertEquals(1, persistedEvents.size());
         EventVO eventVO = persistedEvents.get(0);
-        Assert.assertEquals(eventVO.getType(), eventType);
+        Assert.assertEquals(eventType, eventVO.getType());
         Assert.assertTrue(eventVO.getDescription().endsWith(eventDescription));
-        Assert.assertEquals(eventVO.getLevel(), EventVO.LEVEL_ERROR);
+        Assert.assertEquals(EventVO.LEVEL_ERROR, eventVO.getLevel());
-        Assert.assertEquals(eventVO.getState(), com.cloud.event.Event.State.Completed);
+        Assert.assertEquals(com.cloud.event.Event.State.Completed, eventVO.getState());
     }
 
     @Test
@@ -276,14 +278,14 @@ public class ActionEventInterceptorTest {
         Method m = tester.getClass().getMethod("testMethod");
         actionEventInterceptor.interceptException(m, tester, null);
 
-        Assert.assertEquals(persistedEvents.size(), 1);
+        Assert.assertEquals(1, persistedEvents.size());
         EventVO eventVO = persistedEvents.get(0);
-        Assert.assertEquals(eventVO.getType(), eventType);
+        Assert.assertEquals(eventType, eventVO.getType());
         Assert.assertTrue(eventVO.getDescription().endsWith(eventDescription));
-        Assert.assertEquals(eventVO.getLevel(), EventVO.LEVEL_ERROR);
+        Assert.assertEquals(EventVO.LEVEL_ERROR, eventVO.getLevel());
-        Assert.assertEquals(eventVO.getState(), com.cloud.event.Event.State.Completed);
+        Assert.assertEquals(com.cloud.event.Event.State.Completed, eventVO.getState());
-        Assert.assertEquals(eventVO.getResourceId(), resourceId);
+        Assert.assertEquals(resourceId, eventVO.getResourceId());
-        Assert.assertEquals(eventVO.getResourceType(), resourceType.toString());
+        Assert.assertEquals(resourceType.toString(), eventVO.getResourceType());
     }
 
     @Test

ui/public/config.json

@@ -10,12 +10,12 @@
   "docBase": "http://docs.cloudstack.apache.org/en/latest",
   "appTitle": "CloudStack",
   "footer": "Licensed under the <a href='http://www.apache.org/licenses/' target='_blank'>Apache License</a>, Version 2.0.",
+  "loginTitle": "CloudStack",
+  "loginFavicon": "assets/logo.svg",
   "loginFooter": "",
   "logo": "assets/logo.svg",
   "minilogo": "assets/mini-logo.svg",
   "banner": "assets/banner.svg",
-  "loginPageTitle": "CloudStack",
-  "loginPageFavicon": "assets/logo.svg",
   "error": {
     "403": "assets/403.png",
     "404": "assets/404.png",

ui/public/index.html

@@ -58,8 +58,8 @@
     fetch('./config.json')
       .then(response => response.json())
       .then(data => {
-        document.getElementById("favicon").setAttribute("href", data.loginPageFavicon);
+        document.getElementById("favicon").setAttribute("href", data.loginFavicon);
-        document.getElementById("title").innerHTML = data.loginPageTitle;
+        document.getElementById("title").innerHTML = data.loginTitle;
       }).catch((err) => {});
   </script>
 </html>

ui/public/locales/en.json

@@ -1942,6 +1942,7 @@
 "label.service.connectivity.distributedroutercapabilitycheckbox": "Distributed router",
 "label.service.connectivity.regionlevelvpccapabilitycheckbox": "Region level VPC",
 "label.service.group": "Service group",
+"label.serviceip": "Service IP",
 "label.service.lb.elasticlbcheckbox": "Elastic LB",
 "label.service.lb.inlinemodedropdown": "Mode",
 "label.service.lb.lbisolationdropdown": "LB isolation",

ui/src/components/view/InfoCard.vue

@@ -104,18 +104,6 @@
                   v-if="resource.id"
                 />
               </a-tooltip>
-              <a-tooltip placement="right" >
-                <template #title>
-                  <span>{{ $t('label.copy.consoleurl') }}</span>
-                </template>
-                <console
-                  copyUrlToClipboard
-                  style="margin-top: -5px;"
-                  :resource="resource"
-                  size="default"
-                  v-if="resource.id"
-                />
-              </a-tooltip>
             </div>
           </slot>
         </div>

ui/src/components/widgets/Console.vue

@@ -56,17 +56,10 @@ export default {
         this.url = (json && json.createconsoleendpointresponse) ? json.createconsoleendpointresponse.consoleendpoint.url : '#/exception/404'
         if (json.createconsoleendpointresponse.consoleendpoint.success) {
           if (this.copyUrlToClipboard) {
+            this.$copyText(this.url)
             this.$message.success({
               content: this.$t('label.copied.clipboard')
             })
-            const hiddenElement = document.createElement('textarea')
-            hiddenElement.value = this.url
-            document.body.appendChild(hiddenElement)
-            hiddenElement.focus()
-            hiddenElement.select()
-
-            document.execCommand('copy')
-            document.body.removeChild(hiddenElement)
           } else {
             window.open(this.url, '_blank')
           }

ui/src/config/section/account.js

@@ -31,6 +31,30 @@ export default {
     name: 'accountuser',
     title: 'label.users',
     param: 'account'
+  }, {
+    name: 'vm',
+    title: 'label.vms',
+    param: 'account'
+  }, {
+    name: 'volume',
+    title: 'label.volumes',
+    param: 'account'
+  }, {
+    name: 'guestnetwork',
+    title: 'label.networks',
+    param: 'account'
+  }, {
+    name: 'ssh',
+    title: 'label.sshkeypairs',
+    param: 'account'
+  }, {
+    name: 'userdata',
+    title: 'label.userdata',
+    param: 'account'
+  }, {
+    name: 'template',
+    title: 'label.templates',
+    param: 'account'
   }],
   filters: () => {
     const filters = ['enabled', 'disabled', 'locked']

ui/src/config/section/compute.js

@@ -32,7 +32,7 @@ export default {
       getApiToCall: () => store.getters.metrics ? 'listVirtualMachinesMetrics' : 'listVirtualMachines',
       resourceType: 'UserVm',
       params: () => {
-        var params = { details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp' }
+        var params = { details: 'group,nics,secgrp,tmpl,servoff,diskoff,iso,volume,affgrp,backoff' }
         if (store.getters.metrics) {
           params = { details: 'all,stats' }
         }

ui/src/config/section/domain.js

@@ -32,6 +32,22 @@ export default {
     name: 'account',
     title: 'label.accounts',
     param: 'domainid'
+  }, {
+    name: 'vm',
+    title: 'label.vms',
+    param: 'domainid'
+  }, {
+    name: 'volume',
+    title: 'label.volumes',
+    param: 'domainid'
+  }, {
+    name: 'guestnetwork',
+    title: 'label.networks',
+    param: 'domainid'
+  }, {
+    name: 'template',
+    title: 'label.templates',
+    param: 'domainid'
   }],
   tabs: [
     {

ui/src/config/section/infra/hosts.js

@@ -32,7 +32,7 @@ export default {
   },
   params: { type: 'routing' },
   columns: () => {
-    const fields = ['name', 'state', 'resourcestate', 'ipaddress', 'hypervisor', 'instances', 'powerstate']
+    const fields = ['name', 'state', 'resourcestate', 'ipaddress', 'hypervisor', 'instances', 'powerstate', 'version']
     const metricsFields = ['cpunumber', 'cputotalghz', 'cpuusedghz', 'cpuallocatedghz', 'memorytotalgb', 'memoryusedgb', 'memoryallocatedgb', 'networkread', 'networkwrite']
     if (store.getters.metrics) {
       fields.push(...metricsFields)

ui/src/config/section/infra/managementServers.js

@@ -26,8 +26,8 @@ export default {
   permission: ['listManagementServersMetrics'],
   resourceType: 'ManagementServer',
   columns: () => {
-    const fields = ['name', 'state', 'version']
+    const fields = ['name', 'state', 'serviceip', 'version', 'osdistribution', 'agentcount']
-    const metricsFields = ['collectiontime', 'availableprocessors', 'cpuload', 'heapmemoryused', 'agentcount']
+    const metricsFields = ['collectiontime', 'availableprocessors', 'cpuload', 'heapmemoryused']
     if (store.getters.metrics) {
       fields.push(...metricsFields)
     }