Merge c8159dc928 into bce3e54a7e

Co-authored-by: Daman Arora <daman.arora@shapeblue.com>

api/src/main/java/org/apache/cloudstack/api/command/admin/network/UpdateNetworkOfferingCmd.java

@@ -78,6 +78,7 @@ public class UpdateNetworkOfferingCmd extends BaseCmd {
 
     @Parameter(name = ApiConstants.DOMAIN_ID,
             type = CommandType.STRING,
+            length = 4096,
             description = "The ID of the containing domain(s) as comma separated string, public for public offerings")
     private String domainIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateDiskOfferingCmd.java

@@ -75,6 +75,7 @@ public class UpdateDiskOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/offering/UpdateServiceOfferingCmd.java

@@ -69,6 +69,7 @@ public class UpdateServiceOfferingCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/UpdateVPCOfferingCmd.java

@@ -65,6 +65,7 @@ public class UpdateVPCOfferingCmd extends BaseAsyncCmd {
     @Parameter(name = ApiConstants.ZONE_ID,
             type = CommandType.STRING,
             description = "The ID of the containing zone(s) as comma separated string, all for all zones offerings",
+            length = 4096,
             since = "4.13")
     private String zoneIds;
 

engine/schema/src/main/java/com/cloud/dc/dao/ClusterDao.java

@@ -22,7 +22,6 @@ import java.util.Map;
 import com.cloud.cpu.CPU;
 import com.cloud.dc.ClusterVO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.utils.Pair;
 import com.cloud.utils.db.GenericDao;
 
 public interface ClusterDao extends GenericDao<ClusterVO, Long> {
@@ -34,8 +33,6 @@ public interface ClusterDao extends GenericDao<ClusterVO, Long> {
 
     List<HypervisorType> getAvailableHypervisorInZone(Long zoneId);
 
-    List<Pair<HypervisorType, CPU.CPUArch>> listDistinctHypervisorsArchAcrossClusters(Long zoneId);
-
     List<ClusterVO> listByDcHyType(long dcId, String hyType);
 
     Map<Long, List<Long>> getPodClusterIdMap(List<Long> clusterIds);

engine/schema/src/main/java/com/cloud/dc/dao/ClusterDaoImpl.java

@@ -37,7 +37,6 @@ import com.cloud.dc.HostPodVO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.org.Grouping;
 import com.cloud.org.Managed;
-import com.cloud.utils.Pair;
 import com.cloud.utils.db.GenericDaoBase;
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.JoinBuilder;
@@ -168,22 +167,6 @@ public class ClusterDaoImpl extends GenericDaoBase<ClusterVO, Long> implements C
                 .collect(Collectors.toList());
     }
 
-    @Override
-    public List<Pair<HypervisorType, CPU.CPUArch>> listDistinctHypervisorsArchAcrossClusters(Long zoneId) {
-        SearchBuilder<ClusterVO> sb = createSearchBuilder();
-        sb.select(null, Func.DISTINCT_PAIR, sb.entity().getHypervisorType(), sb.entity().getArch());
-        sb.and("zoneId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ);
-        sb.done();
-        SearchCriteria<ClusterVO> sc = sb.create();
-        if (zoneId != null) {
-            sc.setParameters("zoneId", zoneId);
-        }
-        final List<ClusterVO> clusters = search(sc, null);
-        return clusters.stream()
-                .map(c -> new Pair<>(c.getHypervisorType(), c.getArch()))
-                .collect(Collectors.toList());
-    }
-
     @Override
     public Map<Long, List<Long>> getPodClusterIdMap(List<Long> clusterIds) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();

engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java

@@ -58,10 +58,10 @@ import org.ini4j.Ini;
 
 import com.cloud.cpu.CPU;
 import com.cloud.dc.DataCenterVO;
-import com.cloud.dc.dao.ClusterDao;
-import com.cloud.dc.dao.ClusterDaoImpl;
 import com.cloud.dc.dao.DataCenterDao;
 import com.cloud.dc.dao.DataCenterDaoImpl;
+import com.cloud.host.dao.HostDao;
+import com.cloud.host.dao.HostDaoImpl;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.GuestOSVO;
@@ -108,9 +108,16 @@ public class SystemVmTemplateRegistration {
     private static Integer LINUX_12_ID = 363;
     private static final Integer SCRIPT_TIMEOUT = 1800000;
     private static final Integer LOCK_WAIT_TIMEOUT = 1200;
-    protected static final List<CPU.CPUArch> DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList(
-            CPU.CPUArch.arm64
-    );
+    protected static final List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> BUNDLED_TEMPLATE_HYPERVISOR_ARCH_TYPES =
+            Arrays.asList(
+                    new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64),
+                    new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.amd64),
+                    new Pair<>(Hypervisor.HypervisorType.XenServer, CPU.CPUArch.amd64)
+            );
+    protected static final List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> DOWNLOADABLE_TEMPLATE_HYPERVISOR_ARCH_TYPES =
+            Arrays.asList(
+                    new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.arm64)
+            );
 
     public static String CS_MAJOR_VERSION = null;
     public static String CS_TINY_VERSION = null;
@@ -130,7 +137,7 @@ public class SystemVmTemplateRegistration {
     @Inject
     ImageStoreDetailsDao imageStoreDetailsDao;
     @Inject
-    ClusterDao clusterDao;
+    HostDao hostDao;
     @Inject
     ConfigurationDao configurationDao;
     @Inject
@@ -148,9 +155,9 @@ public class SystemVmTemplateRegistration {
         vmInstanceDao = new VMInstanceDaoImpl();
         imageStoreDao = new ImageStoreDaoImpl();
         imageStoreDetailsDao = new ImageStoreDetailsDaoImpl();
-        clusterDao = new ClusterDaoImpl();
         configurationDao = new ConfigurationDaoImpl();
         guestOSDao = new GuestOSDaoImpl();
+        hostDao = new HostDaoImpl();
         tempDownloadDir = new File(System.getProperty("java.io.tmpdir"));
     }
 
@@ -857,7 +864,9 @@ public class SystemVmTemplateRegistration {
             return templateFile;
         }
         LOGGER.debug("{} is not present", templateFile.getAbsolutePath());
-        if (DOWNLOADABLE_TEMPLATE_ARCH_TYPES.contains(templateDetails.getArch()) &&
+        Pair<Hypervisor.HypervisorType, CPU.CPUArch> templateHypervisorAndArch =
+                new Pair<>(templateDetails.getHypervisorType(), templateDetails.getArch());
+        if (DOWNLOADABLE_TEMPLATE_HYPERVISOR_ARCH_TYPES.contains(templateHypervisorAndArch) &&
                 StringUtils.isNotBlank(templateDetails.getUrl())) {
             LOGGER.debug("Downloading the template file {} for {}",
                     templateDetails.getUrl(), templateDetails.getHypervisorArchLog());
@@ -900,6 +909,11 @@ public class SystemVmTemplateRegistration {
                         matchedTemplate.getHypervisorArchLog());
                 continue;
             }
+            if (!tempFile.exists() && !BUNDLED_TEMPLATE_HYPERVISOR_ARCH_TYPES.contains(hypervisorArch)) {
+                LOGGER.warn("Template for {} not found locally, moving ahead",
+                        matchedTemplate.getHypervisorArchLog());
+                continue;
+            }
             if (isTemplateFileChecksumDifferent(matchedTemplate, tempFile)) {
                 templatesFound = false;
                 break;
@@ -917,7 +931,7 @@ public class SystemVmTemplateRegistration {
         String nfsVersion = getNfsVersion(storeUrlAndId.second());
         mountStore(storeUrlAndId.first(), filePath, nfsVersion);
         List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> hypervisorArchList =
-                clusterDao.listDistinctHypervisorsArchAcrossClusters(zoneId);
+                hostDao.listDistinctHypervisorArchTypes(zoneId);
         for (Pair<Hypervisor.HypervisorType, CPU.CPUArch> hypervisorArch : hypervisorArchList) {
             Hypervisor.HypervisorType hypervisorType = hypervisorArch.first();
             MetadataTemplateDetails templateDetails = getMetadataTemplateDetails(hypervisorType,
@@ -1065,9 +1079,9 @@ public class SystemVmTemplateRegistration {
             public void doInTransactionWithoutResult(final TransactionStatus status) {
                 List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> hypervisorsInUse;
                 try {
-                    hypervisorsInUse = clusterDao.listDistinctHypervisorsArchAcrossClusters(null);
+                    hypervisorsInUse = hostDao.listDistinctHypervisorArchTypes(null);
                 } catch (final Exception e) {
-                    throw new CloudRuntimeException("Exception while getting hypervisor types from clusters", e);
+                    throw new CloudRuntimeException("Exception while getting hypervisor types from hosts", e);
                 }
                 Collection<MetadataTemplateDetails> templateEntries = NewTemplateMap.values();
                 for (MetadataTemplateDetails templateDetails : templateEntries) {

engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDao.java

@@ -31,4 +31,6 @@ public interface VMScheduledJobDao extends GenericDao<VMScheduledJobVO, Long> {
     int expungeJobsForSchedules(List<Long> scheduleId, Date dateAfter);
 
     int expungeJobsBefore(Date currentTimestamp);
+
+    VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp);
 }

engine/schema/src/main/java/org/apache/cloudstack/vm/schedule/dao/VMScheduledJobDaoImpl.java

@@ -39,6 +39,8 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
 
     private final SearchBuilder<VMScheduledJobVO> expungeJobForScheduleSearch;
 
+    private final SearchBuilder<VMScheduledJobVO> scheduleAndTimestampSearch;
+
     static final String SCHEDULED_TIMESTAMP = "scheduled_timestamp";
 
     static final String VM_SCHEDULE_ID = "vm_schedule_id";
@@ -58,6 +60,11 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
         expungeJobForScheduleSearch.and(VM_SCHEDULE_ID, expungeJobForScheduleSearch.entity().getVmScheduleId(), SearchCriteria.Op.IN);
         expungeJobForScheduleSearch.and(SCHEDULED_TIMESTAMP, expungeJobForScheduleSearch.entity().getScheduledTime(), SearchCriteria.Op.GTEQ);
         expungeJobForScheduleSearch.done();
+
+        scheduleAndTimestampSearch = createSearchBuilder();
+        scheduleAndTimestampSearch.and(VM_SCHEDULE_ID, scheduleAndTimestampSearch.entity().getVmScheduleId(), SearchCriteria.Op.EQ);
+        scheduleAndTimestampSearch.and(SCHEDULED_TIMESTAMP, scheduleAndTimestampSearch.entity().getScheduledTime(), SearchCriteria.Op.EQ);
+        scheduleAndTimestampSearch.done();
     }
 
     /**
@@ -92,4 +99,12 @@ public class VMScheduledJobDaoImpl extends GenericDaoBase<VMScheduledJobVO, Long
         sc.setParameters(SCHEDULED_TIMESTAMP, date);
         return expunge(sc);
     }
+
+    @Override
+    public VMScheduledJobVO findByScheduleAndTimestamp(long scheduleId, Date scheduledTimestamp) {
+        SearchCriteria<VMScheduledJobVO> sc = scheduleAndTimestampSearch.create();
+        sc.setParameters(VM_SCHEDULE_ID, scheduleId);
+        sc.setParameters(SCHEDULED_TIMESTAMP, scheduledTimestamp);
+        return findOneBy(sc);
+    }
 }

engine/schema/src/test/java/com/cloud/dc/dao/ClusterDaoImplTest.java

@@ -17,7 +17,6 @@
 package com.cloud.dc.dao;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doReturn;
@@ -37,13 +36,9 @@ import org.mockito.InjectMocks;
 import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
-import com.cloud.cpu.CPU;
 import com.cloud.dc.ClusterVO;
-import com.cloud.hypervisor.Hypervisor;
-import com.cloud.utils.Pair;
 import com.cloud.utils.db.GenericSearchBuilder;
 import com.cloud.utils.db.SearchBuilder;
-import com.cloud.utils.db.SearchCriteria;
 
 @RunWith(MockitoJUnitRunner.class)
 public class ClusterDaoImplTest {
@@ -80,39 +75,4 @@ public class ClusterDaoImplTest {
         verify(clusterDao).customSearch(genericSearchBuilder.create(), null);
         assertTrue(result.isEmpty());
     }
-
-    @Test
-    public void listDistinctHypervisorsArchAcrossClusters_WithZone() {
-        Long zoneId = 123L;
-        ClusterVO cluster1 = mock(ClusterVO.class);
-        when(cluster1.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.XenServer);
-        when(cluster1.getArch()).thenReturn(CPU.CPUArch.amd64);
-        ClusterVO cluster2 = mock(ClusterVO.class);
-        when(cluster2.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM);
-        when(cluster2.getArch()).thenReturn(CPU.CPUArch.arm64);
-        List<ClusterVO> dummyHosts = Arrays.asList(cluster1, cluster2);
-        doReturn(dummyHosts).when(clusterDao).search(any(SearchCriteria.class), isNull());
-        List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> result = clusterDao.listDistinctHypervisorsArchAcrossClusters(zoneId);
-        assertNotNull(result);
-        assertEquals(2, result.size());
-        assertEquals(Hypervisor.HypervisorType.XenServer, result.get(0).first());
-        assertEquals(CPU.CPUArch.amd64, result.get(0).second());
-        assertEquals(Hypervisor.HypervisorType.KVM, result.get(1).first());
-        assertEquals(CPU.CPUArch.arm64, result.get(1).second());
-    }
-
-    @Test
-    public void listDistinctHypervisorsArchAcrossClusters_WithoutZone() {
-        Long zoneId = null;
-        ClusterVO cluster = mock(ClusterVO.class);
-        when(cluster.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.VMware);
-        when(cluster.getArch()).thenReturn(CPU.CPUArch.amd64);
-        List<ClusterVO> dummyHosts = Collections.singletonList(cluster);
-        doReturn(dummyHosts).when(clusterDao).search(any(SearchCriteria.class), isNull());
-        List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> result = clusterDao.listDistinctHypervisorsArchAcrossClusters(zoneId);
-        assertNotNull(result);
-        assertEquals(1, result.size());
-        assertEquals(Hypervisor.HypervisorType.VMware, result.get(0).first());
-        assertEquals(CPU.CPUArch.amd64, result.get(0).second());
-    }
 }

engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java

@@ -53,7 +53,7 @@ import org.mockito.Spy;
 import org.mockito.junit.MockitoJUnitRunner;
 
 import com.cloud.cpu.CPU;
-import com.cloud.dc.dao.ClusterDao;
+import com.cloud.host.dao.HostDao;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.VMTemplateVO;
 import com.cloud.storage.dao.VMTemplateDao;
@@ -67,7 +67,7 @@ import com.cloud.utils.script.Script;
 public class SystemVmTemplateRegistrationTest {
 
     @Mock
-    ClusterDao clusterDao;
+    HostDao hostDao;
 
     @Mock
     VMTemplateDao vmTemplateDao;
@@ -363,14 +363,15 @@ public class SystemVmTemplateRegistrationTest {
         systemVmTemplateRegistration.validateTemplates(list);
     }
 
+    @Test
     public void testValidateTemplates_downloadableFileNotFound() {
-        CPU.CPUArch arch = SystemVmTemplateRegistration.DOWNLOADABLE_TEMPLATE_ARCH_TYPES.get(0);
+        Pair<Hypervisor.HypervisorType, CPU.CPUArch> hypervisorTypeCPUArchPair = SystemVmTemplateRegistration.DOWNLOADABLE_TEMPLATE_HYPERVISOR_ARCH_TYPES.get(0);
         List<Pair<Hypervisor.HypervisorType, CPU.CPUArch>> list = new ArrayList<>();
-        list.add(new Pair<>(Hypervisor.HypervisorType.KVM, arch));
+        list.add(hypervisorTypeCPUArchPair);
         SystemVmTemplateRegistration.MetadataTemplateDetails details =
                 Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class);
         SystemVmTemplateRegistration.NewTemplateMap.put(SystemVmTemplateRegistration.getHypervisorArchKey(
-                Hypervisor.HypervisorType.KVM, arch), details);
+                hypervisorTypeCPUArchPair.first(), hypervisorTypeCPUArchPair.second()), details);
         doReturn(null).when(systemVmTemplateRegistration).getTemplateFile(details);
         systemVmTemplateRegistration.validateTemplates(list);
     }
@@ -404,7 +405,7 @@ public class SystemVmTemplateRegistrationTest {
             Hypervisor.HypervisorType hypervisorType = Hypervisor.HypervisorType.KVM;
             CPU.CPUArch arch = CPU.CPUArch.getDefault();
             hypervisorArchList.add(new Pair<>(hypervisorType, arch));
-            doReturn(hypervisorArchList).when(clusterDao).listDistinctHypervisorsArchAcrossClusters(zoneId);
+            doReturn(hypervisorArchList).when(hostDao).listDistinctHypervisorArchTypes(zoneId);
             SystemVmTemplateRegistration.MetadataTemplateDetails details =
                     Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class);
             String name = "existing";

server/src/main/java/com/cloud/storage/StorageManagerImpl.java

@@ -3653,7 +3653,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                             DataStoreRole.Image, store.getId());
                     if (CollectionUtils.isEmpty(stores)) {
                         List<Pair<HypervisorType, CPU.CPUArch>> hypervisorTypes =
-                                _clusterDao.listDistinctHypervisorsArchAcrossClusters(zoneId);
+                                _hostDao.listDistinctHypervisorArchTypes(zoneId);
                         TransactionLegacy txn = TransactionLegacy.open("AutomaticTemplateRegister");
                         SystemVmTemplateRegistration systemVmTemplateRegistration = new SystemVmTemplateRegistration();
                         String filePath = null;
@@ -3676,7 +3676,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
                                 }
                             }
                         } catch (Exception e) {
-                            logger.error("Failed to register systemVM template(s)");
+                            logger.error("Failed to register systemVM template(s)", e);
                         } finally {
                             SystemVmTemplateRegistration.unmountStore(filePath);
                             txn.close();

server/src/main/java/org/apache/cloudstack/vm/schedule/VMSchedulerImpl.java

@@ -162,7 +162,13 @@ public class VMSchedulerImpl extends ManagerBase implements VMScheduler, Configu
         }
 
         Date scheduledDateTime = Date.from(ts.toInstant());
-        VMScheduledJobVO scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime);
+        VMScheduledJobVO scheduledJob = vmScheduledJobDao.findByScheduleAndTimestamp(vmSchedule.getId(), scheduledDateTime);
+        if (scheduledJob != null) {
+            logger.trace("Job is already scheduled for schedule {} at {}", vmSchedule, scheduledDateTime);
+            return scheduledDateTime;
+        }
+
+        scheduledJob = new VMScheduledJobVO(vmSchedule.getVmId(), vmSchedule.getId(), vmSchedule.getAction(), scheduledDateTime);
         try {
             vmScheduledJobDao.persist(scheduledJob);
             ActionEventUtils.onScheduledActionEvent(User.UID_SYSTEM, vm.getAccountId(), actionEventMap.get(vmSchedule.getAction()),

ui/src/utils/plugins.js

@@ -218,18 +218,19 @@ export const notifierPlugin = {
         if (error.response.status) {
           msg = `${i18n.global.t('message.request.failed')} (${error.response.status})`
         }
-        if (error.message) {
-          desc = error.message
-        }
-        if (error.response.headers && 'x-description' in error.response.headers) {
+        if (error.response.headers?.['x-description']) {
           desc = error.response.headers['x-description']
-        }
-        if (desc === '' && error.response.data) {
+        } else if (error.response.data) {
           const responseKey = _.findKey(error.response.data, 'errortext')
           if (responseKey) {
             desc = error.response.data[responseKey].errortext
+          } else if (typeof error.response.data === 'string') {
+            desc = error.response.data
           }
         }
+        if (!desc && error.message) {
+          desc = error.message
+        }
       }
       let countNotify = store.getters.countNotify
       countNotify++

ui/src/views/image/RegisterOrUploadTemplate.vue

@@ -638,11 +638,7 @@ export default {
         this.$emit('refresh-data')
         this.closeAction()
       }).catch(e => {
-        this.$notification.error({
-          message: this.$t('message.upload.failed'),
-          description: `${this.$t('message.upload.template.failed.description')} -  ${e}`,
-          duration: 0
-        })
+        this.$notifyError(e)
       })
     },
     fetchCustomHypervisorName () {