From 1a2dbebe4894f4db7e2d3529fd56befe9ef7dfd3 Mon Sep 17 00:00:00 2001 From: dahn Date: Wed, 15 Nov 2023 09:48:11 +0100 Subject: [PATCH 01/50] Let Prometheus exporter plugin support utf8 characters (#8228) --- .../metrics/PrometheusExporterServerImpl.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java index b5ac137dadd..cc3b7d55653 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java @@ -28,6 +28,7 @@ import javax.inject.Inject; import java.io.IOException; import java.io.OutputStream; import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; import java.util.Arrays; public class PrometheusExporterServerImpl extends ManagerBase implements PrometheusExporterServer, Configurable { @@ -57,11 +58,21 @@ public class PrometheusExporterServerImpl extends ManagerBase implements Prometh response = prometheusExporter.getMetrics(); responseCode = 200; } - httpExchange.getResponseHeaders().set("content-type", "text/plain"); - httpExchange.sendResponseHeaders(responseCode, response.length()); + byte[] bytesToOutput = response.getBytes(StandardCharsets.UTF_8); + httpExchange.getResponseHeaders().set("content-type", "text/plain; charset=UTF-8"); + httpExchange.sendResponseHeaders(responseCode, bytesToOutput.length); final OutputStream os = httpExchange.getResponseBody(); - os.write(response.getBytes()); - os.close(); + try { + os.write(bytesToOutput); + } catch (IOException e) { + LOG.error(String.format("could not export Prometheus data due to %s", e.getLocalizedMessage())); + if (LOG.isDebugEnabled()) { + LOG.debug("Error during Prometheus export: ", e); + } + os.write("The system could not export Prometheus due to an internal error. Contact your operator to learn about the reason.".getBytes()); + } finally { + os.close(); + } } } From 1b56a8e8733a4057f65a6496c67e0d2e97cc68ff Mon Sep 17 00:00:00 2001 From: Oleg Chuev <91337563+OlegChuev@users.noreply.github.com> Date: Sat, 18 Nov 2023 19:36:49 +0200 Subject: [PATCH 02/50] Fixed spelling and added missing states to response (#8248) --- .../cloudstack/api/response/AcquireIPAddressResponse.java | 2 +- .../org/apache/cloudstack/api/response/IPAddressResponse.java | 2 +- .../org/apache/cloudstack/api/response/PortableIpResponse.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java index a74c95780d1..7270fa949c8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/AcquireIPAddressResponse.java @@ -124,7 +124,7 @@ public class AcquireIPAddressResponse extends BaseResponse implements Controlle private String networkId; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated and Releasing") private String state; @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java index 13497d89308..4188543af36 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/IPAddressResponse.java @@ -124,7 +124,7 @@ public class IPAddressResponse extends BaseResponseWithAnnotations implements Co private String networkId; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing, Reserved and Free") private String state; @SerializedName(ApiConstants.PHYSICAL_NETWORK_ID) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java index 73008b01ccd..e477b111561 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/PortableIpResponse.java @@ -68,7 +68,7 @@ public class PortableIpResponse extends BaseResponse { private Date allocated; @SerializedName(ApiConstants.STATE) - @Param(description = "State of the ip address. Can be: Allocatin, Allocated and Releasing") + @Param(description = "State of the ip address. Can be: Allocating, Allocated, Releasing and Free") private String state; public void setRegionId(Integer regionId) { From 5c7e4b7edc88a588b94db64998e83b6fcc76048a Mon Sep 17 00:00:00 2001 From: Abhisar Sinha <63767682+abh1sar@users.noreply.github.com> Date: Sun, 19 Nov 2023 11:50:57 +0530 Subject: [PATCH 03/50] api: add ipaddress argument to disassociateIPAddress (#8222) This PR adds argument 'ipadress' to the disassociateIpAddress api. IP address can be disassociated by directly giving the address instead of ID. Fixes: #8125 --- .../com/cloud/network/NetworkService.java | 2 + .../user/address/DisassociateIPAddrCmd.java | 68 ++++++++++++++----- .../com/cloud/network/NetworkServiceImpl.java | 5 ++ .../com/cloud/vpc/MockNetworkManagerImpl.java | 9 +++ test/integration/smoke/test_network.py | 62 ++++++++++++++++- tools/marvin/marvin/lib/base.py | 9 ++- 6 files changed, 137 insertions(+), 18 deletions(-) diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index d959e3abce0..82d229da459 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -114,6 +114,8 @@ public interface NetworkService { IpAddress getIp(long id); + IpAddress getIp(String ipAddress); + Network updateGuestNetwork(final UpdateNetworkCmd cmd); /** diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java index b31520e1b88..f9bfcb253b4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/address/DisassociateIPAddrCmd.java @@ -46,10 +46,14 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, required = true, description = "the ID of the public IP address" - + " to disassociate") + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = IPAddressResponse.class, description = "the ID of the public IP address" + + " to disassociate. Mutually exclusive with the ipaddress parameter") private Long id; + @Parameter(name=ApiConstants.IP_ADDRESS, type=CommandType.STRING, since="4.19.0", description="IP Address to be disassociated." + + " Mutually exclusive with the id parameter") + private String ipAddress; + // unexposed parameter needed for events logging @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, expose = false) private Long ownerId; @@ -59,7 +63,18 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// public Long getIpAddressId() { - return id; + if (id != null & ipAddress != null) { + throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter"); + } + + if (id != null) { + return id; + } else if (ipAddress != null) { + IpAddress ip = getIpAddressByIp(ipAddress); + return ip.getId(); + } + + throw new InvalidParameterValueException("Please specify either IP address or IP address ID"); } ///////////////////////////////////////////////////// @@ -68,12 +83,13 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public void execute() throws InsufficientAddressCapacityException { - CallContext.current().setEventDetails("IP ID: " + getIpAddressId()); + Long ipAddressId = getIpAddressId(); + CallContext.current().setEventDetails("IP ID: " + ipAddressId); boolean result = false; - if (!isPortable(id)) { - result = _networkService.releaseIpAddress(getIpAddressId()); + if (!isPortable()) { + result = _networkService.releaseIpAddress(ipAddressId); } else { - result = _networkService.releasePortableIpAddress(getIpAddressId()); + result = _networkService.releasePortableIpAddress(ipAddressId); } if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); @@ -85,7 +101,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public String getEventType() { - if (!isPortable(id)) { + if (!isPortable()) { return EventTypes.EVENT_NET_IP_RELEASE; } else { return EventTypes.EVENT_PORTABLE_IP_RELEASE; @@ -100,10 +116,7 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public long getEntityOwnerId() { if (ownerId == null) { - IpAddress ip = getIpAddress(id); - if (ip == null) { - throw new InvalidParameterValueException("Unable to find IP address by ID=" + id); - } + IpAddress ip = getIpAddress(); ownerId = ip.getAccountId(); } @@ -120,11 +133,11 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { @Override public Long getSyncObjId() { - IpAddress ip = getIpAddress(id); + IpAddress ip = getIpAddress(); return ip.getAssociatedWithNetworkId(); } - private IpAddress getIpAddress(long id) { + private IpAddress getIpAddressById(Long id) { IpAddress ip = _entityMgr.findById(IpAddress.class, id); if (ip == null) { @@ -134,6 +147,29 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { } } + private IpAddress getIpAddressByIp(String ipAddress) { + IpAddress ip = _networkService.getIp(ipAddress); + if (ip == null) { + throw new InvalidParameterValueException("Unable to find IP address by IP address=" + ipAddress); + } else { + return ip; + } + } + + private IpAddress getIpAddress() { + if (id != null & ipAddress != null) { + throw new InvalidParameterValueException("id parameter is mutually exclusive with ipaddress parameter"); + } + + if (id != null) { + return getIpAddressById(id); + } else if (ipAddress != null){ + return getIpAddressByIp(ipAddress); + } + + throw new InvalidParameterValueException("Please specify either IP address or IP address ID"); + } + @Override public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.IpAddress; @@ -144,8 +180,8 @@ public class DisassociateIPAddrCmd extends BaseAsyncCmd { return getIpAddressId(); } - private boolean isPortable(long id) { - IpAddress ip = getIpAddress(id); + private boolean isPortable() { + IpAddress ip = getIpAddress(); return ip.isPortable(); } } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 0c17efaab36..f9f80cb5b76 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -2798,6 +2798,11 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C return _ipAddressDao.findById(ipAddressId); } + @Override + public IpAddress getIp(String ipAddress) { + return _ipAddressDao.findByIp(ipAddress); + } + protected boolean providersConfiguredForExternalNetworking(Collection providers) { for (String providerStr : providers) { Provider provider = Network.Provider.getProvider(providerStr); diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 945448f7173..7535841974e 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -275,6 +275,15 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return null; } + /* (non-Javadoc) + * @see com.cloud.network.NetworkService#getIp(String) + */ + @Override + public IpAddress getIp(String ipAddress) { + // TODO Auto-generated method stub + return null; + } + @Override public Network updateGuestNetwork(final UpdateNetworkCmd cmd) { // TODO Auto-generated method stub diff --git a/test/integration/smoke/test_network.py b/test/integration/smoke/test_network.py index 56b434f3241..8f3f4f533dd 100644 --- a/test/integration/smoke/test_network.py +++ b/test/integration/smoke/test_network.py @@ -871,7 +871,7 @@ class TestReleaseIP(cloudstackTestCase): @attr(tags=["advanced", "advancedns", "smoke", "dvs"], required_hardware="false") def test_releaseIP(self): - """Test for release public IP address""" + """Test for release public IP address using the ID""" logger.debug("Deleting Public IP : %s" % self.ip_addr.id) @@ -930,6 +930,66 @@ class TestReleaseIP(cloudstackTestCase): ) return + @attr(tags=["advanced", "advancedns", "smoke", "dvs"], required_hardware="false") + def test_releaseIP_using_IP(self): + """Test for release public IP address using the address""" + + logger.debug("Deleting Public IP : %s" % self.ip_addr.ipaddress) + self.ip_address.delete_by_ip(self.apiclient) + + retriesCount = 10 + isIpAddressDisassociated = False + while retriesCount > 0: + listResponse = list_publicIP( + self.apiclient, + id=self.ip_addr.id, + state="Allocated" + ) + if listResponse is None: + isIpAddressDisassociated = True + break + retriesCount -= 1 + time.sleep(60) + # End while + + self.assertTrue( + isIpAddressDisassociated, + "Failed to disassociate IP address") + + # ListPortForwardingRules should not list + # associated rules with Public IP address + try: + list_nat_rule = list_nat_rules( + self.apiclient, + id=self.nat_rule.id + ) + logger.debug("List NAT Rule response" + str(list_nat_rule)) + except CloudstackAPIException: + logger.debug("Port Forwarding Rule is deleted") + + # listLoadBalancerRules should not list + # associated rules with Public IP address + try: + list_lb_rule = list_lb_rules( + self.apiclient, + id=self.lb_rule.id + ) + logger.debug("List LB Rule response" + str(list_lb_rule)) + except CloudstackAPIException: + logger.debug("Port Forwarding Rule is deleted") + + # SSH Attempt though public IP should fail + with self.assertRaises(Exception): + SshClient( + self.ip_addr.ipaddress, + self.services["natrule"]["publicport"], + self.virtual_machine.username, + self.virtual_machine.password, + retries=2, + delay=0 + ) + return + class TestDeleteAccount(cloudstackTestCase): diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index 32b0e980407..a9827dcfaaf 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -1879,12 +1879,19 @@ class PublicIPAddress: return PublicIPAddress(apiclient.associateIpAddress(cmd).__dict__) def delete(self, apiclient): - """Dissociate Public IP address""" + """Dissociate Public IP address using the given ID""" cmd = disassociateIpAddress.disassociateIpAddressCmd() cmd.id = self.ipaddress.id apiclient.disassociateIpAddress(cmd) return + def delete_by_ip(self, apiclient): + """Dissociate Public IP address using the given IP address""" + cmd = disassociateIpAddress.disassociateIpAddressCmd() + cmd.ipaddress = self.ipaddress.ipaddress + apiclient.disassociateIpAddress(cmd) + return + @classmethod def list(cls, apiclient, **kwargs): """List all Public IPs matching criteria""" From 3c7c75bacfd5292488e0cd5e235bfb83d06c815a Mon Sep 17 00:00:00 2001 From: anniejili <47866640+anniejili@users.noreply.github.com> Date: Tue, 21 Nov 2023 02:11:04 -0800 Subject: [PATCH 04/50] Clear pool id if volume allocation fails (#8202) * clear pool id if volume allocation fails and leave volume state as Allocated with a pool id assigned * clear_pool_id_if_volume_allocation_fails --------- Co-authored-by: Annie Li --- .../deploy/DeploymentPlanningManagerImpl.java | 10 ++ .../DeploymentPlanningManagerImplTest.java | 103 ++++++++++++++++++ .../cloud/storage/StorageManagerImplTest.java | 30 +++++ 3 files changed, 143 insertions(+) diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index cf4e7fdb2cc..0ef462bb96c 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -1651,6 +1651,16 @@ StateListener, Configurable { for (VolumeVO toBeCreated : volumesTobeCreated) { s_logger.debug("Checking suitable pools for volume (Id, Type): (" + toBeCreated.getId() + "," + toBeCreated.getVolumeType().name() + ")"); + if (toBeCreated.getState() == Volume.State.Allocated && toBeCreated.getPoolId() != null) { + toBeCreated.setPoolId(null); + if (!_volsDao.update(toBeCreated.getId(), toBeCreated)) { + throw new CloudRuntimeException(String.format("Error updating volume [%s] to clear pool Id.", toBeCreated.getId())); + } + if (s_logger.isDebugEnabled()) { + String msg = String.format("Setting pool_id to NULL for volume id=%s as it is in Allocated state", toBeCreated.getId()); + s_logger.debug(msg); + } + } // If the plan specifies a poolId, it means that this VM's ROOT // volume is ready and the pool should be reused. // In this case, also check if rest of the volumes are ready and can diff --git a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java index d79010ee5c0..ea9e2bb2090 100644 --- a/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java +++ b/server/src/test/java/com/cloud/deploy/DeploymentPlanningManagerImplTest.java @@ -19,6 +19,8 @@ package com.cloud.deploy; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import java.io.IOException; import java.util.ArrayList; @@ -33,6 +35,7 @@ import javax.naming.ConfigurationException; import com.cloud.dc.ClusterDetailsVO; import com.cloud.dc.DataCenter; +import com.cloud.dc.HostPodVO; import com.cloud.gpu.GPU; import com.cloud.host.Host; import com.cloud.host.HostVO; @@ -40,19 +43,26 @@ import com.cloud.host.Status; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; +import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.Pair; +import com.cloud.vm.DiskProfile; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.Type; import com.cloud.vm.VirtualMachineProfile; import com.cloud.vm.VirtualMachineProfileImpl; import org.apache.cloudstack.affinity.dao.AffinityGroupDomainMapDao; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.commons.collections.CollectionUtils; import org.junit.Assert; @@ -67,6 +77,7 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; +import org.powermock.api.mockito.PowerMockito; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan.Filter; @@ -197,6 +208,18 @@ public class DeploymentPlanningManagerImplTest { @Mock ConfigurationDao configDao; + @Mock + AccountManager _accountMgr; + + @Inject + DiskOfferingDao _diskOfferingDao; + + @Mock + DataStoreManager _dataStoreManager; + + @Inject + HostPodDao _podDao; + private static final long dataCenterId = 1L; private static final long instanceId = 123L; private static final long hostId = 0L; @@ -242,6 +265,8 @@ public class DeploymentPlanningManagerImplTest { List planners = new ArrayList(); planners.add(_planner); _dpm.setPlanners(planners); + StoragePoolAllocator allocator = Mockito.mock(StoragePoolAllocator.class); + _dpm.setStoragePoolAllocators(Arrays.asList(allocator)); Mockito.when(host.getId()).thenReturn(hostId); Mockito.doNothing().when(_dpm).avoidDisabledResources(vmProfile, dc, avoids); @@ -702,6 +727,84 @@ public class DeploymentPlanningManagerImplTest { } } + @Test + public void findSuitablePoolsForVolumesTest() throws Exception { + Long diskOfferingId = 1L; + HostVO host = Mockito.spy(new HostVO("host")); + Map hostDetails = new HashMap<>() { + { + put(Host.HOST_VOLUME_ENCRYPTION, "true"); + } + }; + host.setDetails(hostDetails); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + + VolumeVO vol1 = Mockito.spy(new VolumeVO("vol1", dataCenterId, podId, 1L, 1L, instanceId, "folder", "path", + Storage.ProvisioningType.THIN, (long) 10 << 30, Volume.Type.ROOT)); + Mockito.when(vol1.getId()).thenReturn(1L); + vol1.setState(Volume.State.Allocated); + vol1.setPassphraseId(1L); + vol1.setPoolId(1L); + vol1.setDiskOfferingId(diskOfferingId); + + StoragePoolVO storagePool = new StoragePoolVO(); + storagePool.setStatus(StoragePoolStatus.Maintenance); + storagePool.setId(vol1.getPoolId()); + storagePool.setDataCenterId(dataCenterId); + storagePool.setPodId(podId); + storagePool.setClusterId(clusterId); + + DiskProfile diskProfile = Mockito.mock(DiskProfile.class); + + StoragePoolAllocator allocator = Mockito.mock(StoragePoolAllocator.class); + + DataCenterDeployment plan = new DataCenterDeployment(dataCenterId, podId, clusterId, null, null, null); + + Account account = Mockito.mock(Account.class); + Mockito.when(account.getId()).thenReturn(1L); + Mockito.when(vmProfile.getOwner()).thenReturn(account); + Mockito.when(_accountMgr.isRootAdmin(account.getId())).thenReturn(Boolean.FALSE); + + Mockito.when(_dcDao.findById(dataCenterId)).thenReturn(dc); + Mockito.when(dc.getAllocationState()).thenReturn(AllocationState.Enabled); + + HostPodVO podVo = Mockito.mock(HostPodVO.class); + Mockito.when(podVo.getAllocationState()).thenReturn(AllocationState.Enabled); + Mockito.doReturn(podVo).when(_podDao).findById(podId); + + ClusterVO cluster = Mockito.mock(ClusterVO.class); + Mockito.when(cluster.getAllocationState()).thenReturn(AllocationState.Enabled); + Mockito.when(_clusterDao.findById(clusterId)).thenReturn(cluster); + + DiskOfferingVO diskOffering = Mockito.mock(DiskOfferingVO.class); + + Mockito.when(_diskOfferingDao.findById(vol1.getDiskOfferingId())).thenReturn(diskOffering); + VirtualMachineTemplate vmt = Mockito.mock(VirtualMachineTemplate.class); + + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(vmProfile.getServiceOffering()).thenReturn(serviceOffering); + + PrimaryDataStore primaryDataStore = Mockito.mock(PrimaryDataStore.class); + + Mockito.when(vmt.getFormat()).thenReturn(Storage.ImageFormat.ISO); + Mockito.when(vmProfile.getTemplate()).thenReturn(vmt); + + Mockito.when(vmProfile.getId()).thenReturn(1L); + Mockito.when(vmProfile.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(volDao.findUsableVolumesForInstance(1L)).thenReturn(Arrays.asList(vol1)); + Mockito.when(volDao.findByInstanceAndType(1L, Volume.Type.ROOT)).thenReturn(Arrays.asList(vol1)); + Mockito.when(_dataStoreManager.getPrimaryDataStore(vol1.getPoolId())).thenReturn((DataStore) primaryDataStore); + Mockito.when(avoids.shouldAvoid(storagePool)).thenReturn(Boolean.FALSE); + PowerMockito.whenNew(DiskProfile.class).withAnyArguments().thenReturn(diskProfile); + + Mockito.doReturn(Arrays.asList(storagePool)).when(allocator).allocateToPool(diskProfile, vmProfile, plan, + avoids, 10); + Mockito.when(volDao.update(vol1.getId(), vol1)).thenReturn(true); + _dpm.findSuitablePoolsForVolumes(vmProfile, plan, avoids, 10); + verify(vol1, times(1)).setPoolId(null); + assertTrue(vol1.getPoolId() == null); + + } // This is so ugly but everything is so intertwined... private DeploymentClusterPlanner setupMocksForPlanDeploymentHostTests(HostVO host, VolumeVO vol1) { long diskOfferingId = 345L; diff --git a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java index e7004ba7c5d..20e1be95e52 100644 --- a/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java +++ b/server/src/test/java/com/cloud/storage/StorageManagerImplTest.java @@ -19,6 +19,8 @@ package com.cloud.storage; import java.util.ArrayList; import java.util.List; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -128,4 +130,32 @@ public class StorageManagerImplTest { Assert.assertTrue(storageManagerImpl.isVolumeSuspectedDestroyDuplicateOfVmVolume(volume)); } + @Test + public void storagePoolCompatibleWithVolumePoolTestVolumeWithPoolIdInAllocatedState() { + StoragePoolVO storagePool = new StoragePoolVO(); + storagePool.setPoolType(Storage.StoragePoolType.PowerFlex); + storagePool.setId(1L); + VolumeVO volume = new VolumeVO(); + volume.setState(Volume.State.Allocated); + volume.setPoolId(1L); + PrimaryDataStoreDao storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); + storageManagerImpl._storagePoolDao = storagePoolDao; + Mockito.doReturn(storagePool).when(storagePoolDao).findById(volume.getPoolId()); + Assert.assertFalse(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume)); + + } + + @Test + public void storagePoolCompatibleWithVolumePoolTestVolumeWithoutPoolIdInAllocatedState() { + StoragePoolVO storagePool = new StoragePoolVO(); + storagePool.setPoolType(Storage.StoragePoolType.PowerFlex); + storagePool.setId(1L); + VolumeVO volume = new VolumeVO(); + volume.setState(Volume.State.Allocated); + PrimaryDataStoreDao storagePoolDao = Mockito.mock(PrimaryDataStoreDao.class); + storageManagerImpl._storagePoolDao = storagePoolDao; + Assert.assertTrue(storageManagerImpl.storagePoolCompatibleWithVolumePool(storagePool, volume)); + + } + } From 8f390873772ef59c3ba81c837fffe707ebd80e72 Mon Sep 17 00:00:00 2001 From: slavkap <51903378+slavkap@users.noreply.github.com> Date: Fri, 24 Nov 2023 10:47:08 +0200 Subject: [PATCH 05/50] Fix typo in ssvm check script (#8191) --- systemvm/agent/scripts/ssvm-check.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/systemvm/agent/scripts/ssvm-check.sh b/systemvm/agent/scripts/ssvm-check.sh index b2721a93b3f..f5d69cb4548 100644 --- a/systemvm/agent/scripts/ssvm-check.sh +++ b/systemvm/agent/scripts/ssvm-check.sh @@ -103,10 +103,10 @@ else echo "Verifying if we can at least ping the storage" STORAGE_ADDRESSES=`grep "secondaryStorageServerAddress" $CMDLINE | sed -E 's/.*secondaryStorageServerAddress=([^ ]*).*/\1/g'` - if [[ -z "$STORAGE_ADDRESS" ]] + if [[ -z "$STORAGE_ADDRESSES" ]] then STORAGE_NETWORK_GATEWAY=`grep "storagegateway" $CMDLINE | sed -E 's/.*storagegateway=([^ ]*).*/\1/g'` - echo "Storage address is empty, trying to ping storage network gateway instead ($STORAGE_NETWORK_GATEWAY)" + echo "Storage address list is empty, trying to ping storage network gateway instead ($STORAGE_NETWORK_GATEWAY)" ping -c 2 $STORAGE_NETWORK_GATEWAY if [ $? -eq 0 ] then @@ -118,7 +118,7 @@ else fi else echo "Storage address(s): $STORAGE_ADDRESSES, trying to ping" - STORAGE_ADDRESS_LIST=$(echo $STORAGE_ADDRESSES | tr ",") + STORAGE_ADDRESS_LIST=$(echo $STORAGE_ADDRESSES | tr "," "\n") for STORAGE_ADDRESS in $STORAGE_ADDRESS_LIST do echo "Pinging storage address: $STORAGE_ADDRESS" From f0b757e91ea3ebaf1000b93c10462eda6b367509 Mon Sep 17 00:00:00 2001 From: StepSecurity Bot Date: Sun, 26 Nov 2023 22:37:19 -0800 Subject: [PATCH 06/50] [StepSecurity] ci: Harden GitHub Actions (#8209) Signed-off-by: StepSecurity Bot --- .github/workflows/build.yml | 3 +++ .github/workflows/ci.yml | 3 +++ .github/workflows/rat.yml | 3 +++ .github/workflows/ui.yml | 3 +++ 4 files changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 457dd2e1af4..1be892f4577 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0422cd4bd0..d6b5f5a1c4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: if: github.repository == 'apache/cloudstack' diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index d243fa863fe..64fa4c3da0c 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 280024b5a91..4d89977adf9 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -23,6 +23,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: build: runs-on: ubuntu-22.04 From b0a9949775b47bb589837dda40d9869aab01dff1 Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Tue, 28 Nov 2023 13:32:44 +0530 Subject: [PATCH 07/50] ui: include domain and account field for createTemplate forms (#8257) This PR includes Domain and Account parameters in CreateTemplate form in UI for Admin account. It will expose optional parameters domainId and account in UI supported by createTemplate API #8210 --- ui/src/config/section/storage.js | 9 ++- ui/src/views/storage/CreateTemplate.vue | 92 ++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 269c1f81788..3c2bd30db9b 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -253,7 +253,14 @@ export default { ((record.type === 'ROOT' && record.vmstate === 'Stopped') || (record.type !== 'ROOT' && !record.virtualmachineid && !['Allocated', 'Uploaded'].includes(record.state))) }, - args: ['volumeid', 'name', 'displaytext', 'ostypeid', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled'], + args: (record, store) => { + var fields = ['volumeid', 'name', 'displaytext', 'ostypeid', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled'] + if (['Admin', 'DomainAdmin'].includes(store.userInfo.roletype)) { + fields.push('domainid') + fields.push('account') + } + return fields + }, mapping: { volumeid: { value: (record) => { return record.id } diff --git a/ui/src/views/storage/CreateTemplate.vue b/ui/src/views/storage/CreateTemplate.vue index 294abe330cb..13ce75777fb 100644 --- a/ui/src/views/storage/CreateTemplate.vue +++ b/ui/src/views/storage/CreateTemplate.vue @@ -69,6 +69,48 @@ + + + + + + + + {{ opt.path || opt.name || opt.description }} + + + + + + + + + {{ acc.name }} + + + + { + this.domains = json.listdomainsresponse.domain + }).finally(() => { + this.domainLoading = false + this.handleDomainChange(null) + }) + }, + handleDomainChange (domain) { + this.domainid = domain + this.form.account = null + this.account = null + if ('listAccounts' in this.$store.getters.apis) { + this.fetchAccounts() + } + }, + fetchAccounts () { + api('listAccounts', { + domainid: this.domainid + }).then(response => { + this.accounts = response.listaccountsresponse.account || [] + }).catch(error => { + this.$notifyError(error) + }) + }, + handleAccountChange (acc) { + if (acc) { + this.account = acc.name + } else { + this.account = acc + } + }, handleSubmit (e) { if (this.loading) return this.formRef.value.validate().then(() => { @@ -277,6 +364,9 @@ export default { }, closeModal () { this.$emit('close-action') + }, + isAdminOrDomainAdmin () { + return ['Admin', 'DomainAdmin'].includes(this.$store.getters.userInfo.roletype) } } } From 60b399f8758dd6e057fa887317e44c3b0791a0a1 Mon Sep 17 00:00:00 2001 From: sato03 Date: Tue, 28 Nov 2023 05:13:18 -0300 Subject: [PATCH 08/50] Fix create private gateway rollback (#8244) When creating a private gateway, if an ACL verification error occurs, the changes made up to that point are not rolled back, resulting in inconsistent records in the database. This PR intends to fix this bug and, if an error occurs during the creation of the private gateway, the changes will be rolled back. --- .../com/cloud/network/vpc/VpcManagerImpl.java | 39 +++++++++++++------ .../cloud/network/vpc/VpcManagerImplTest.java | 23 +++++++++++ 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java index b4cb73f035d..6bc70a9cef4 100644 --- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.network.vpc; +import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -2161,6 +2162,8 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis final PhysicalNetwork physNetFinal = physNet; VpcGatewayVO gatewayVO = null; try { + validateVpcPrivateGatewayAclId(vpcId, aclId); + s_logger.debug("Creating Private gateway for VPC " + vpc); // 1) create private network unless it is existing and // lswitch'd @@ -2200,18 +2203,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis _dcDao.update(dc.getId(), dc); } - long networkAclId = NetworkACL.DEFAULT_DENY; - if (aclId != null) { - final NetworkACLVO aclVO = _networkAclDao.findById(aclId); - if (aclVO == null) { - throw new InvalidParameterValueException("Invalid network acl id passed "); - } - if (aclVO.getVpcId() != vpcId && !(aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW)) { - throw new InvalidParameterValueException("Private gateway and network acl are not in the same vpc"); - } - - networkAclId = aclId; - } + Long networkAclId = ObjectUtils.defaultIfNull(aclId, NetworkACL.DEFAULT_DENY); { // experimental block, this is a hack // set vpc id in network to null @@ -2244,6 +2236,29 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis return getVpcPrivateGateway(gatewayVO.getId()); } + /** + * This method checks if the ACL that is being used to create the private gateway is valid. First, the aclId is used to search for a {@link NetworkACLVO} object + * by calling the {@link NetworkACLDao#findById(Serializable)} method. If the object is null, an {@link InvalidParameterValueException} exception is thrown. + * Secondly, we check if the ACL and the private gateway are in the same VPC and an {@link InvalidParameterValueException} is thrown if they are not. + * + * @param vpcId Private gateway VPC ID. + * @param aclId Private gateway ACL ID. + * @throws InvalidParameterValueException + */ + protected void validateVpcPrivateGatewayAclId(long vpcId, Long aclId) { + if (aclId == null) { + return; + } + + final NetworkACLVO aclVO = _networkAclDao.findById(aclId); + if (aclVO == null) { + throw new InvalidParameterValueException("Invalid network acl id passed."); + } + if (aclVO.getVpcId() != vpcId && !(aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW)) { + throw new InvalidParameterValueException("Private gateway and network acl are not in the same vpc."); + } + } + private void validateVpcPrivateGatewayAssociateNetworkId(NetworkOfferingVO ntwkOff, String broadcastUri, Long associatedNetworkId, Boolean bypassVlanOverlapCheck) { // Validate vlanId and associatedNetworkId if (broadcastUri == null && associatedNetworkId == null) { diff --git a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java index b802d1fb355..deffb165f29 100644 --- a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java @@ -47,6 +47,7 @@ import com.cloud.network.element.NetworkElement; import com.cloud.network.router.CommandSetupHelper; import com.cloud.network.router.NetworkHelper; import com.cloud.network.router.VirtualRouter; +import com.cloud.network.vpc.dao.NetworkACLDao; import com.cloud.network.vpc.dao.VpcDao; import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.network.vpc.dao.VpcOfferingServiceMapDao; @@ -127,6 +128,8 @@ public class VpcManagerImplTest { @Mock NetworkDao networkDao; @Mock + NetworkACLDao networkACLDaoMock; + @Mock NetworkModel networkModel; @Mock NetworkOfferingServiceMapDao networkOfferingServiceMapDao; @@ -150,6 +153,8 @@ public class VpcManagerImplTest { NetworkService networkServiceMock; @Mock FirewallRulesDao firewallDao; + @Mock + NetworkACLVO networkACLVOMock; public static final long ACCOUNT_ID = 1; private AccountVO account; @@ -168,6 +173,9 @@ public class VpcManagerImplTest { final Long vpcOwnerId = 1L; final String vpcName = "Test-VPC"; final String vpcDomain = "domain"; + final Long aclId = 1L; + final Long differentVpcAclId = 3L; + final Long vpcId = 1L; private AutoCloseable closeable; @@ -203,6 +211,7 @@ public class VpcManagerImplTest { manager._dcDao = dataCenterDao; manager._ntwkSvc = networkServiceMock; manager._firewallDao = firewallDao; + manager._networkAclDao = networkACLDaoMock; CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); registerCallContext(); overrideDefaultConfigValue(NetworkService.AllowUsersToSpecifyVRMtu, "_defaultValue", "false"); @@ -487,4 +496,18 @@ public class VpcManagerImplTest { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } } + + @Test + public void validateVpcPrivateGatewayAclIdTestNullAclVoThrowsInvalidParameterValueException() { + Mockito.doReturn(null).when(networkACLDaoMock).findById(aclId); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateVpcPrivateGatewayAclId(vpcId, aclId)); + } + + @Test + public void validateVpcPrivateGatewayTestAclFromDifferentVpcThrowsInvalidParameterValueException() { + Mockito.doReturn(2L).when(networkACLVOMock).getVpcId(); + Mockito.doReturn(networkACLVOMock).when(networkACLDaoMock).findById(differentVpcAclId); + Assert.assertThrows(InvalidParameterValueException.class, () -> manager.validateVpcPrivateGatewayAclId(vpcId, differentVpcAclId)); + } + } From 956efb27d9307e7145afab02a93f751f8772ef83 Mon Sep 17 00:00:00 2001 From: "Rodrigo D. Lopez" <19981369+RodrigoDLopez@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:50:37 -0300 Subject: [PATCH 09/50] allow filtering of listDiskOffering and listServiceOffering APIs by account or project (#7082) --- .../user/offering/ListDiskOfferingsCmd.java | 4 +-- .../offering/ListServiceOfferingsCmd.java | 4 +-- .../com/cloud/api/query/QueryManagerImpl.java | 35 +++++++++++-------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java index 5fa24ec1630..5ab675ae435 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListDiskOfferingsCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.offering; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -23,14 +24,13 @@ import org.apache.log4j.Logger; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListDomainResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.ListResponse; @APICommand(name = "listDiskOfferings", description = "Lists all available disk offerings.", responseObject = DiskOfferingResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListDiskOfferingsCmd extends BaseListDomainResourcesCmd { +public class ListDiskOfferingsCmd extends BaseListProjectAndAccountResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListDiskOfferingsCmd.class.getName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java index 3208ef58a4f..cb155d24ad8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/offering/ListServiceOfferingsCmd.java @@ -16,12 +16,12 @@ // under the License. package org.apache.cloudstack.api.command.user.offering; +import org.apache.cloudstack.api.BaseListProjectAndAccountResourcesCmd; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.log4j.Logger; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseListDomainResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; @@ -29,7 +29,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; @APICommand(name = "listServiceOfferings", description = "Lists all available service offerings.", responseObject = ServiceOfferingResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) -public class ListServiceOfferingsCmd extends BaseListDomainResourcesCmd { +public class ListServiceOfferingsCmd extends BaseListProjectAndAccountResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListServiceOfferingsCmd.class.getName()); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index ce9252fb045..51aed5af66c 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -2929,6 +2929,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Object id = cmd.getId(); Object keyword = cmd.getKeyword(); Long domainId = cmd.getDomainId(); + Long projectId = cmd.getProjectId(); + String accountName = cmd.getAccountName(); Boolean isRootAdmin = _accountMgr.isRootAdmin(account.getAccountId()); Boolean isRecursive = cmd.isRecursive(); Long zoneId = cmd.getZoneId(); @@ -2938,7 +2940,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // Keeping this logic consistent with domain specific zones // if a domainId is provided, we just return the disk offering // associated with this domain - if (domainId != null) { + if (domainId != null && accountName == null) { if (_accountMgr.isRootAdmin(account.getId()) || isPermissible(account.getDomainId(), domainId)) { // check if the user's domain == do's domain || user's domain is // a child of so's domain for non-root users @@ -3019,9 +3021,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // Filter offerings that are not associated with caller's domain // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! - Account caller = CallContext.current().getCallingAccount(); - if (caller.getType() != Account.Type.ADMIN) { - Domain callerDomain = _domainDao.findById(caller.getDomainId()); + account = _accountMgr.finalizeOwner(account, accountName, domainId, projectId); + if (!Account.Type.ADMIN.equals(account.getType())) { + Domain callerDomain = _domainDao.findById(account.getDomainId()); List domainIds = findRelatedDomainIds(callerDomain, isRecursive); List ids = _diskOfferingDetailsDao.findOfferingIdsByDomainIds(domainIds); @@ -3100,6 +3102,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q searchFilter.addOrderBy(ServiceOfferingJoinVO.class, "id", true); Account caller = CallContext.current().getCallingAccount(); + Long projectId = cmd.getProjectId(); + String accountName = cmd.getAccountName(); Object name = cmd.getServiceOfferingName(); Object id = cmd.getId(); Object keyword = cmd.getKeyword(); @@ -3115,9 +3119,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Integer cpuSpeed = cmd.getCpuSpeed(); Boolean encryptRoot = cmd.getEncryptRoot(); + final Account owner = _accountMgr.finalizeOwner(caller, accountName, domainId, projectId); SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); if (!_accountMgr.isRootAdmin(caller.getId()) && isSystem) { - throw new InvalidParameterValueException("Only ROOT admins can access system's offering"); + throw new InvalidParameterValueException("Only ROOT admins can access system offerings."); } // Keeping this logic consistent with domain specific zones @@ -3126,8 +3131,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (domainId != null && !_accountMgr.isRootAdmin(caller.getId())) { // check if the user's domain == so's domain || user's domain is a // child of so's domain - if (!isPermissible(caller.getDomainId(), domainId)) { - throw new PermissionDeniedException("The account:" + caller.getAccountName() + " does not fall in the same domain hierarchy as the service offering"); + if (!isPermissible(owner.getDomainId(), domainId)) { + throw new PermissionDeniedException("The account:" + owner.getAccountName() + " does not fall in the same domain hierarchy as the service offering"); } } @@ -3139,7 +3144,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q throw ex; } - _accountMgr.checkAccess(caller, null, true, vmInstance); + _accountMgr.checkAccess(owner, null, true, vmInstance); currentVmOffering = _srvOfferingDao.findByIdIncludingRemoved(vmInstance.getId(), vmInstance.getServiceOfferingId()); if (! currentVmOffering.isDynamic()) { @@ -3187,22 +3192,22 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } // boolean includePublicOfferings = false; - if ((_accountMgr.isNormalUser(caller.getId()) || _accountMgr.isDomainAdmin(caller.getId())) || caller.getType() == Account.Type.RESOURCE_DOMAIN_ADMIN) { + if ((_accountMgr.isNormalUser(owner.getId()) || _accountMgr.isDomainAdmin(owner.getId())) || owner.getType() == Account.Type.RESOURCE_DOMAIN_ADMIN) { // For non-root users. if (isSystem) { throw new InvalidParameterValueException("Only root admins can access system's offering"); } if (isRecursive) { // domain + all sub-domains - if (caller.getType() == Account.Type.NORMAL) { + if (owner.getType() == Account.Type.NORMAL) { throw new InvalidParameterValueException("Only ROOT admins and Domain admins can list service offerings with isrecursive=true"); } } } else { // for root users - if (caller.getDomainId() != 1 && isSystem) { // NON ROOT admin - throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering"); + if (owner.getDomainId() != 1 && isSystem) { // NON ROOT admin + throw new InvalidParameterValueException("Non ROOT admins cannot access system's offering."); } - if (domainId != null) { + if (domainId != null && accountName == null) { sc.addAnd("domainId", Op.FIND_IN_SET, String.valueOf(domainId)); } } @@ -3286,8 +3291,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q // Filter offerings that are not associated with caller's domain // Fetch the offering ids from the details table since theres no smart way to filter them in the join ... yet! - if (caller.getType() != Account.Type.ADMIN) { - Domain callerDomain = _domainDao.findById(caller.getDomainId()); + if (owner.getType() != Account.Type.ADMIN) { + Domain callerDomain = _domainDao.findById(owner.getDomainId()); List domainIds = findRelatedDomainIds(callerDomain, isRecursive); List ids = _srvOfferingDetailsDao.findOfferingIdsByDomainIds(domainIds); From cb2b6aca45c81de19a5d9f1bf31a2f022d7f8c31 Mon Sep 17 00:00:00 2001 From: Wei Zhou Date: Wed, 29 Nov 2023 18:55:26 +0100 Subject: [PATCH 10/50] server: check if there are active nics before network GC (#8204) --- .../main/java/com/cloud/vm/dao/NicDao.java | 4 ++- .../java/com/cloud/vm/dao/NicDaoImpl.java | 35 +++++++++++++------ .../com/cloud/network/NetworkModelImpl.java | 9 ++--- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index c52c690d8b5..13eb04ba6b8 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -79,7 +79,9 @@ public interface NicDao extends GenericDao { List listByNetworkIdTypeAndGatewayAndBroadcastUri(long networkId, VirtualMachine.Type vmType, String gateway, URI broadcastUri); - int countNicsForStartingVms(long networkId); + int countNicsForNonStoppedVms(long networkId); + + int countNicsForNonStoppedRunningVrs(long networkId); NicVO getControlNicForVM(long vmId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index c8efc074a10..fdc36b4f918 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -44,7 +44,7 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { private GenericSearchBuilder IpSearch; private SearchBuilder NonReleasedSearch; private GenericSearchBuilder deviceIdSearch; - private GenericSearchBuilder CountByForStartingVms; + private GenericSearchBuilder CountByForNonStoppedVms; private SearchBuilder PeerRouterSearch; @Inject @@ -91,14 +91,16 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { deviceIdSearch.and("instance", deviceIdSearch.entity().getInstanceId(), Op.EQ); deviceIdSearch.done(); - CountByForStartingVms = createSearchBuilder(Integer.class); - CountByForStartingVms.select(null, Func.COUNT, CountByForStartingVms.entity().getId()); - CountByForStartingVms.and("networkId", CountByForStartingVms.entity().getNetworkId(), Op.EQ); - CountByForStartingVms.and("removed", CountByForStartingVms.entity().getRemoved(), Op.NULL); + CountByForNonStoppedVms = createSearchBuilder(Integer.class); + CountByForNonStoppedVms.select(null, Func.COUNT, CountByForNonStoppedVms.entity().getId()); + CountByForNonStoppedVms.and("vmType", CountByForNonStoppedVms.entity().getVmType(), Op.EQ); + CountByForNonStoppedVms.and("vmTypeNEQ", CountByForNonStoppedVms.entity().getVmType(), Op.NEQ); + CountByForNonStoppedVms.and("networkId", CountByForNonStoppedVms.entity().getNetworkId(), Op.EQ); + CountByForNonStoppedVms.and("removed", CountByForNonStoppedVms.entity().getRemoved(), Op.NULL); SearchBuilder join1 = _vmDao.createSearchBuilder(); - join1.and("state", join1.entity().getState(), Op.EQ); - CountByForStartingVms.join("vm", join1, CountByForStartingVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); - CountByForStartingVms.done(); + join1.and("state", join1.entity().getState(), Op.IN); + CountByForNonStoppedVms.join("vm", join1, CountByForNonStoppedVms.entity().getInstanceId(), join1.entity().getId(), JoinBuilder.JoinType.INNER); + CountByForNonStoppedVms.done(); PeerRouterSearch = createSearchBuilder(); PeerRouterSearch.and("instanceId", PeerRouterSearch.entity().getInstanceId(), Op.NEQ); @@ -338,10 +340,21 @@ public class NicDaoImpl extends GenericDaoBase implements NicDao { } @Override - public int countNicsForStartingVms(long networkId) { - SearchCriteria sc = CountByForStartingVms.create(); + public int countNicsForNonStoppedVms(long networkId) { + SearchCriteria sc = CountByForNonStoppedVms.create(); sc.setParameters("networkId", networkId); - sc.setJoinParameters("vm", "state", VirtualMachine.State.Starting); + sc.setParameters("vmType", VirtualMachine.Type.User); + sc.setJoinParameters("vm", "state", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Running, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating}); + List results = customSearch(sc, null); + return results.get(0); + } + + @Override + public int countNicsForNonStoppedRunningVrs(long networkId) { + SearchCriteria sc = CountByForNonStoppedVms.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("vmTypeNEQ", VirtualMachine.Type.User); + sc.setJoinParameters("vm", "state", new Object[] {VirtualMachine.State.Starting, VirtualMachine.State.Stopping, VirtualMachine.State.Migrating}); List results = customSearch(sc, null); return results.get(0); } diff --git a/server/src/main/java/com/cloud/network/NetworkModelImpl.java b/server/src/main/java/com/cloud/network/NetworkModelImpl.java index beb416cab57..696e93d999f 100644 --- a/server/src/main/java/com/cloud/network/NetworkModelImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkModelImpl.java @@ -2559,10 +2559,11 @@ public class NetworkModelImpl extends ManagerBase implements NetworkModel, Confi return false; } - //if the network has vms in Starting state (nics for those might not be allocated yet as Starting state also used when vm is being Created) - //don't GC - if (_nicDao.countNicsForStartingVms(networkId) > 0) { - s_logger.debug("Network id=" + networkId + " is not ready for GC as it has vms that are Starting at the moment"); + // if the network has user vms in Starting/Stopping/Migrating/Running state, or VRs in Starting/Stopping/Migrating state, don't GC + // The active nics count (nics_count in op_networks table) might be wrong due to some reasons, should check the state of vms as well. + // (nics for Starting VMs might not be allocated yet as Starting state also used when vm is being Created) + if (_nicDao.countNicsForNonStoppedVms(networkId) > 0 || _nicDao.countNicsForNonStoppedRunningVrs(networkId) > 0) { + s_logger.debug("Network id=" + networkId + " is not ready for GC as it has vms that are not Stopped at the moment"); return false; } From 26b01f6f3be3f68dc0fd7ab6bade5129e9905f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Jandre?= <48719461+JoaoJandre@users.noreply.github.com> Date: Thu, 30 Nov 2023 05:36:47 -0300 Subject: [PATCH 11/50] Flexible tags for hosts and storage pools (#7489) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: João Jandre --- .../apache/cloudstack/api/ApiConstants.java | 4 + .../api/command/admin/host/UpdateHostCmd.java | 7 + .../admin/storage/CreateStoragePoolCmd.java | 7 + .../admin/storage/UpdateStoragePoolCmd.java | 7 + .../cloudstack/api/response/HostResponse.java | 12 ++ .../api/response/StoragePoolResponse.java | 12 ++ .../storage/PrimaryDataStoreParameters.java | 10 ++ .../main/java/com/cloud/host/HostTagVO.java | 16 ++ .../src/main/java/com/cloud/host/HostVO.java | 24 ++- .../main/java/com/cloud/host/dao/HostDao.java | 6 + .../java/com/cloud/host/dao/HostDaoImpl.java | 103 +++++++++++-- .../java/com/cloud/host/dao/HostTagsDao.java | 10 +- .../com/cloud/host/dao/HostTagsDaoImpl.java | 39 +++-- .../com/cloud/storage/StoragePoolTagVO.java | 28 ++++ .../cloud/storage/dao/StoragePoolTagsDao.java | 6 +- .../storage/dao/StoragePoolTagsDaoImpl.java | 25 ++- .../datastore/db/PrimaryDataStoreDao.java | 8 +- .../datastore/db/PrimaryDataStoreDaoImpl.java | 46 +++++- .../META-INF/db/schema-41810to41900.sql | 9 ++ .../META-INF/db/views/cloud.host_view.sql | 111 ++++++++++++++ .../db/views/cloud.storage_pool_view.sql | 68 ++++++++ .../test/java/com/cloud/host/HostVOTest.java | 145 ++++++++++-------- .../AbstractStoragePoolAllocator.java | 4 + .../ClusterScopeStoragePoolAllocator.java | 6 +- .../allocator/LocalStoragePoolAllocator.java | 5 +- .../ZoneWideStoragePoolAllocator.java | 8 +- .../datastore/PrimaryDataStoreHelper.java | 2 +- .../activationrule/presetvariables/Host.java | 10 ++ .../presetvariables/PresetVariableHelper.java | 26 +++- .../presetvariables/Storage.java | 11 ++ .../PresetVariableHelperTest.java | 83 +++++++++- .../allocator/impl/RandomAllocator.java | 15 +- .../metrics/PrometheusExporterImpl.java | 9 +- .../ElastistorPrimaryDataStoreLifeCycle.java | 2 + .../DateraPrimaryDataStoreLifeCycle.java | 2 + ...oudStackPrimaryDataStoreLifeCycleImpl.java | 1 + .../LinstorPrimaryDataStoreLifeCycleImpl.java | 2 + .../NexentaPrimaryDataStoreLifeCycle.java | 2 + .../ScaleIOPrimaryDataStoreLifeCycle.java | 2 + .../SolidFirePrimaryDataStoreLifeCycle.java | 2 + ...idFireSharedPrimaryDataStoreLifeCycle.java | 2 + .../allocator/impl/FirstFitAllocator.java | 11 +- .../cloud/api/query/dao/HostJoinDaoImpl.java | 1 + .../api/query/dao/StoragePoolJoinDao.java | 3 + .../api/query/dao/StoragePoolJoinDaoImpl.java | 58 +++++++ .../com/cloud/api/query/vo/HostJoinVO.java | 7 + .../cloud/api/query/vo/StoragePoolJoinVO.java | 7 + .../ConfigurationManagerImpl.java | 45 ++++-- .../deploy/DeploymentPlanningManagerImpl.java | 4 +- .../com/cloud/deploy/FirstFitPlanner.java | 7 + .../cloud/resource/ResourceManagerImpl.java | 22 +-- .../RollingMaintenanceManagerImpl.java | 7 +- .../com/cloud/storage/StorageManagerImpl.java | 30 ++-- .../cloud/storage/VolumeApiServiceImpl.java | 44 ++++-- .../storage/listener/StoragePoolMonitor.java | 2 +- .../ConfigurationManagerTest.java | 10 +- .../storage/VolumeApiServiceImplTest.java | 89 ++++++++--- .../listener/StoragePoolMonitorTest.java | 4 +- ui/public/locales/en.json | 1 + ui/public/locales/pt_BR.json | 1 + ui/src/config/section/infra/hosts.js | 2 +- .../config/section/infra/primaryStorages.js | 2 +- .../utils/jsinterpreter/TagAsRuleHelper.java | 51 ++++++ 63 files changed, 1092 insertions(+), 213 deletions(-) create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql create mode 100644 engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/TagAsRuleHelper.java diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 5b6647991ef..4b437516bd8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1066,6 +1066,10 @@ public class ApiConstants { public static final String CLIENT_ID = "clientid"; public static final String REDIRECT_URI = "redirecturi"; + public static final String IS_TAG_A_RULE = "istagarule"; + + public static final String PARAMETER_DESCRIPTION_IS_TAG_A_RULE = "Whether the informed tag is a JS interpretable rule or not."; + /** * This enum specifies IO Drivers, each option controls specific policies on I/O. * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0). diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java index e3ff130e2d4..9cf47a9c4b9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/UpdateHostCmd.java @@ -60,6 +60,9 @@ public class UpdateHostCmd extends BaseCmd { @Parameter(name = ApiConstants.HOST_TAGS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "list of tags to be added to the host") private List hostTags; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the new uri for the secondary storage: nfs://host/path") private String url; @@ -90,6 +93,10 @@ public class UpdateHostCmd extends BaseCmd { return hostTags; } + public Boolean getIsTagARule() { + return isTagARule; + } + public String getUrl() { return url; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java index 0eacc5cda6b..477d7570dfa 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java @@ -90,6 +90,9 @@ public class CreateStoragePoolCmd extends BaseCmd { description = "hypervisor type of the hosts in zone that will be attached to this storage pool. KVM, VMware supported as of now.") private String hypervisor; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -146,6 +149,10 @@ public class CreateStoragePoolCmd extends BaseCmd { return hypervisor; } + public Boolean isTagARule() { + return this.isTagARule; + } + @Override public long getEntityOwnerId() { return Account.ACCOUNT_ID_SYSTEM; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java index 9e34684a09d..09ec5394921 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateStoragePoolCmd.java @@ -61,6 +61,9 @@ public class UpdateStoragePoolCmd extends BaseCmd { " enable it back.") private Boolean enabled; + @Parameter(name = ApiConstants.IS_TAG_A_RULE, type = CommandType.BOOLEAN, description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -89,6 +92,10 @@ public class UpdateStoragePoolCmd extends BaseCmd { return enabled; } + public Boolean isTagARule() { + return isTagARule; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index e1f1e5ee241..d72d23b99c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -221,6 +221,10 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "comma-separated list of tags for the host") private String hostTags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @SerializedName("hasenoughcapacity") @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise") private Boolean hasEnoughCapacity; @@ -732,4 +736,12 @@ public class HostResponse extends BaseResponseWithAnnotations { public void setEncryptionSupported(Boolean encryptionSupported) { this.encryptionSupported = encryptionSupported; } + + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean tagARule) { + isTagARule = tagARule; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index 89256c26473..183290ec9eb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -101,6 +101,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "the tags for the storage pool") private String tags; + @SerializedName(ApiConstants.IS_TAG_A_RULE) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_IS_TAG_A_RULE) + private Boolean isTagARule; + @SerializedName(ApiConstants.STATE) @Param(description = "the state of the storage pool") private StoragePoolStatus state; @@ -304,6 +308,14 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { this.tags = tags; } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean tagARule) { + isTagARule = tagARule; + } + public StoragePoolStatus getState() { return state; } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java index 1dbff59a891..1b18264df15 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreParameters.java @@ -43,6 +43,8 @@ public class PrimaryDataStoreParameters { private boolean managed; private Long capacityIops; + private Boolean isTagARule; + /** * @return the userInfo */ @@ -277,4 +279,12 @@ public class PrimaryDataStoreParameters { public void setUsedBytes(long usedBytes) { this.usedBytes = usedBytes; } + + public Boolean isTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + } } diff --git a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java index 6b8b754849c..cd4ac29738d 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostTagVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostTagVO.java @@ -24,6 +24,7 @@ import javax.persistence.Id; import javax.persistence.Table; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.BooleanUtils; @Entity @Table(name = "host_tags") @@ -39,12 +40,22 @@ public class HostTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + protected HostTagVO() { } public HostTagVO(long hostId, String tag) { this.hostId = hostId; this.tag = tag; + this.isTagARule = false; + } + + public HostTagVO(long hostId, String tag, Boolean isTagARule) { + this.hostId = hostId; + this.tag = tag; + this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false); } public long getHostId() { @@ -59,6 +70,11 @@ public class HostTagVO implements InternalIdentity { this.tag = tag; } + public boolean getIsTagARule() { + return isTagARule; + } + + @Override public long getId() { return id; diff --git a/engine/schema/src/main/java/com/cloud/host/HostVO.java b/engine/schema/src/main/java/com/cloud/host/HostVO.java index d6e7ea1fd87..697401ad069 100644 --- a/engine/schema/src/main/java/com/cloud/host/HostVO.java +++ b/engine/schema/src/main/java/com/cloud/host/HostVO.java @@ -39,6 +39,7 @@ import javax.persistence.TemporalType; import javax.persistence.Transient; import com.cloud.agent.api.VgpuTypesInfo; +import com.cloud.host.dao.HostTagsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.offering.ServiceOffering; import com.cloud.resource.ResourceState; @@ -46,7 +47,10 @@ import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.NumbersUtil; import com.cloud.utils.db.GenericDao; import java.util.Arrays; + +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang3.StringUtils; @Entity @@ -159,6 +163,14 @@ public class HostVO implements Host { @Transient List hostTags; + /** + * This is a delayed load value. + * If the value is null, then this field has not been loaded yet. + * Call host dao to load it. + */ + @Transient + Boolean isTagARule; + // This value is only for saving and current cannot be loaded. @Transient HashMap> groupDetails = new HashMap>(); @@ -322,8 +334,13 @@ public class HostVO implements Host { return hostTags; } - public void setHostTags(List hostTags) { + public void setHostTags(List hostTags, Boolean isTagARule) { this.hostTags = hostTags; + this.isTagARule = isTagARule; + } + + public Boolean getIsTagARule() { + return isTagARule; } public HashMap> getGpuGroupDetails() { @@ -748,6 +765,11 @@ public class HostVO implements Host { if (serviceOffering == null) { return false; } + + if (BooleanUtils.isTrue(this.getIsTagARule())) { + return TagAsRuleHelper.interpretTagAsRule(this.getHostTags().get(0), serviceOffering.getHostTag(), HostTagsDao.hostTagRuleExecutionTimeout.value()); + } + if (StringUtils.isEmpty(serviceOffering.getHostTag())) { return true; } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index ca51ad3f428..fe30722feb1 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -108,6 +108,8 @@ public interface HostDao extends GenericDao, StateDao listAllHostsByZoneAndHypervisorType(long zoneId, HypervisorType hypervisorType); + List listAllHostsThatHaveNoRuleTag(Host.Type type, Long clusterId, Long podId, Long dcId); + List listAllHostsByType(Host.Type type); HostVO findByPublicIp(String publicIp); @@ -161,4 +163,8 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); + + List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + + List findClustersThatMatchHostTagRule(String computeOfferingTags); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 392b623299f..136351527f6 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -22,9 +22,11 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TimeZone; import java.util.stream.Collectors; @@ -32,6 +34,8 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.persistence.TableGenerator; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import com.cloud.agent.api.VgpuTypesInfo; @@ -80,8 +84,8 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao private static final Logger state_logger = Logger.getLogger(ResourceState.class); private static final String LIST_HOST_IDS_BY_COMPUTETAGS = "SELECT filtered.host_id, COUNT(filtered.tag) AS tag_count " - + "FROM (SELECT host_id, tag FROM host_tags GROUP BY host_id,tag) AS filtered " - + "WHERE tag IN(%s) " + + "FROM (SELECT host_id, tag, is_tag_a_rule FROM host_tags GROUP BY host_id,tag) AS filtered " + + "WHERE tag IN(%s) AND is_tag_a_rule = 0 " + "GROUP BY host_id " + "HAVING tag_count = %s "; private static final String SEPARATOR = ","; @@ -148,6 +152,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected GenericSearchBuilder AllClustersSearch; protected SearchBuilder HostsInClusterSearch; + protected SearchBuilder searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag; + + protected SearchBuilder searchBuilderFindByRuleTag; + protected Attribute _statusAttr; protected Attribute _resourceStateAttr; protected Attribute _msIdAttr; @@ -455,6 +463,22 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao HostIdSearch.and("dataCenterId", HostIdSearch.entity().getDataCenterId(), Op.EQ); HostIdSearch.done(); + searchBuilderFindByRuleTag = _hostTagsDao.createSearchBuilder(); + searchBuilderFindByRuleTag.and("is_tag_a_rule", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.EQ); + searchBuilderFindByRuleTag.or("tagDoesNotExist", searchBuilderFindByRuleTag.entity().getIsTagARule(), Op.NULL); + + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag = createSearchBuilder(); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("type", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getType(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("cluster_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getClusterId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("pod_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getPodId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.and("data_center_id", searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getDataCenterId(), Op.EQ); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.join("id", searchBuilderFindByRuleTag, searchBuilderFindByRuleTag.entity().getHostId(), + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.entity().getId(), JoinType.LEFTOUTER); + + searchBuilderFindByRuleTag.done(); + searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.done(); + _statusAttr = _allAttributes.get("status"); _msIdAttr = _allAttributes.get("managementServerId"); _pingTimeAttr = _allAttributes.get("lastPinged"); @@ -792,9 +816,12 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public List listAllUpAndEnabledNonHAHosts(Type type, Long clusterId, Long podId, long dcId, String haTag) { - SearchBuilder hostTagSearch = null; + SearchBuilder hostTagSearch = _hostTagsDao.createSearchBuilder(); + hostTagSearch.and(); + hostTagSearch.op("isTagARule", hostTagSearch.entity().getIsTagARule(), Op.EQ); + hostTagSearch.or("tagDoesNotExist", hostTagSearch.entity().getIsTagARule(), Op.NULL); + hostTagSearch.cp(); if (haTag != null && !haTag.isEmpty()) { - hostTagSearch = _hostTagsDao.createSearchBuilder(); hostTagSearch.and().op("tag", hostTagSearch.entity().getTag(), SearchCriteria.Op.NEQ); hostTagSearch.or("tagNull", hostTagSearch.entity().getTag(), SearchCriteria.Op.NULL); hostTagSearch.cp(); @@ -809,12 +836,14 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao hostSearch.and("status", hostSearch.entity().getStatus(), SearchCriteria.Op.EQ); hostSearch.and("resourceState", hostSearch.entity().getResourceState(), SearchCriteria.Op.EQ); - if (haTag != null && !haTag.isEmpty()) { - hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER); - } + + hostSearch.join("hostTagSearch", hostTagSearch, hostSearch.entity().getId(), hostTagSearch.entity().getHostId(), JoinBuilder.JoinType.LEFTOUTER); + SearchCriteria sc = hostSearch.create(); + sc.setJoinParameters("hostTagSearch", "isTagARule", false); + if (haTag != null && !haTag.isEmpty()) { sc.setJoinParameters("hostTagSearch", "tag", haTag); } @@ -846,8 +875,13 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao @Override public void loadHostTags(HostVO host) { - List hostTags = _hostTagsDao.getHostTags(host.getId()); - host.setHostTags(hostTags); + List hostTagVOList = _hostTagsDao.getHostTags(host.getId()); + if (CollectionUtils.isNotEmpty(hostTagVOList)) { + List hostTagList = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); + host.setHostTags(hostTagList, hostTagVOList.get(0).getIsTagARule()); + } else { + host.setHostTags(null, null); + } } @DB @@ -881,10 +915,10 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao protected void saveHostTags(HostVO host) { List hostTags = host.getHostTags(); - if (hostTags == null || (hostTags != null && hostTags.isEmpty())) { + if (CollectionUtils.isEmpty(hostTags)) { return; } - _hostTagsDao.persist(host.getId(), hostTags); + _hostTagsDao.persist(host.getId(), hostTags, host.getIsTagARule()); } protected void saveGpuRecords(HostVO host) { @@ -1244,6 +1278,26 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listAllHostsThatHaveNoRuleTag(Type type, Long clusterId, Long podId, Long dcId) { + SearchCriteria sc = searchBuilderFindByIdTypeClusterIdPodIdDcIdAndWithoutRuleTag.create(); + if (type != null) { + sc.setParameters("type", type); + } + if (clusterId != null) { + sc.setParameters("cluster_id", clusterId); + } + if (podId != null) { + sc.setParameters("pod_id", podId); + } + if (dcId != null) { + sc.setParameters("data_center_id", dcId); + } + sc.setJoinParameters("id", "is_tag_a_rule", false); + + return search(sc, null); + } + @Override public List listClustersByHostTag(String computeOfferingTags) { TransactionLegacy txn = TransactionLegacy.currentTxn(); @@ -1266,9 +1320,6 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao result.add(rs.getLong(1)); } pstmt.close(); - if(result.isEmpty()){ - throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags); - } return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + sql, e); @@ -1293,15 +1344,33 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao result.add(rs.getLong(1)); } pstmt.close(); - if(result.isEmpty()){ - throw new CloudRuntimeException("No suitable host found for follow compute offering tags: " + computeOfferingTags); - } return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + select, e); } } + public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + List hostTagVOList = _hostTagsDao.findHostRuleTags(); + List result = new ArrayList<>(); + for (HostTagVO rule: hostTagVOList) { + if (TagAsRuleHelper.interpretTagAsRule(rule.getTag(), computeOfferingTags, HostTagsDao.hostTagRuleExecutionTimeout.value())) { + result.add(findById(rule.getHostId())); + } + } + + return result; + } + + public List findClustersThatMatchHostTagRule(String computeOfferingTags) { + Set result = new HashSet<>(); + List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + for (HostVO host: hosts) { + result.add(host.getClusterId()); + } + return new ArrayList<>(result); + } + private String getHostIdsByComputeTags(List offeringTags){ List questionMarks = new ArrayList(); offeringTags.forEach((tag) -> { questionMarks.add("?"); }); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 0fb5370d81a..d134db33403 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -20,15 +20,21 @@ import java.util.List; import com.cloud.host.HostTagVO; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.framework.config.ConfigKey; public interface HostTagsDao extends GenericDao { - void persist(long hostId, List hostTags); + ConfigKey hostTagRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "host.tag.rule.execution.timeout", "2000", "The maximum runtime, in milliseconds, " + + "to execute a host tag rule; if it is reached, a timeout will happen.", true); - List getHostTags(long hostId); + void persist(long hostId, List hostTags, Boolean isTagARule); + + List getHostTags(long hostId); List getDistinctImplicitHostTags(List hostIds, String[] implicitHostTags); void deleteTags(long hostId); + List findHostRuleTags(); + } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index a73899b7b33..65deb1d1c9b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -16,10 +16,10 @@ // under the License. package com.cloud.host.dao; -import java.util.ArrayList; import java.util.List; - +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import org.springframework.stereotype.Component; import com.cloud.host.HostTagVO; @@ -31,13 +31,14 @@ import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Func; @Component -public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao { +public class HostTagsDaoImpl extends GenericDaoBase implements HostTagsDao, Configurable { protected final SearchBuilder HostSearch; protected final GenericSearchBuilder DistinctImplictTagsSearch; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ); + HostSearch.and("isTagARule", HostSearch.entity().getIsTagARule(), SearchCriteria.Op.EQ); HostSearch.done(); DistinctImplictTagsSearch = createSearchBuilder(String.class); @@ -48,17 +49,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override - public List getHostTags(long hostId) { + public List getHostTags(long hostId) { SearchCriteria sc = HostSearch.create(); sc.setParameters("hostId", hostId); - List results = search(sc, null); - List hostTags = new ArrayList(results.size()); - for (HostTagVO result : results) { - hostTags.add(result.getTag()); - } - - return hostTags; + return search(sc, null); } @Override @@ -80,7 +75,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements } @Override - public void persist(long hostId, List hostTags) { + public List findHostRuleTags() { + SearchCriteria sc = HostSearch.create(); + sc.setParameters("isTagARule", true); + + return search(sc, null); + } + + @Override + public void persist(long hostId, List hostTags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); @@ -91,10 +94,20 @@ public class HostTagsDaoImpl extends GenericDaoBase implements for (String tag : hostTags) { tag = tag.trim(); if (tag.length() > 0) { - HostTagVO vo = new HostTagVO(hostId, tag); + HostTagVO vo = new HostTagVO(hostId, tag, isTagARule); persist(vo); } } txn.commit(); } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {hostTagRuleExecutionTimeout}; + } + + @Override + public String getConfigComponentName() { + return HostTagsDaoImpl.class.getSimpleName(); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java b/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java index 18c0dc3c326..2675c36f27f 100755 --- a/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/StoragePoolTagVO.java @@ -23,7 +23,9 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import com.cloud.utils.NumbersUtil; import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.BooleanUtils; @Entity @Table(name = "storage_pool_tags") @@ -43,9 +45,19 @@ public class StoragePoolTagVO implements InternalIdentity { @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + public StoragePoolTagVO(long poolId, String tag) { this.poolId = poolId; this.tag = tag; + this.isTagARule = false; + } + + public StoragePoolTagVO(long poolId, String tag, Boolean isTagARule) { + this.poolId = poolId; + this.tag = tag; + this.isTagARule = BooleanUtils.toBooleanDefaultIfNull(isTagARule, false); } @Override @@ -61,4 +73,20 @@ public class StoragePoolTagVO implements InternalIdentity { return tag; } + public boolean isTagARule() { + return this.isTagARule; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof StoragePoolTagVO) { + return this.poolId == ((StoragePoolTagVO)obj).getPoolId(); + } + return false; + } + + @Override + public int hashCode() { + return NumbersUtil.hash(id); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java index 946b46b5cfe..9352ee21858 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDao.java @@ -25,10 +25,14 @@ import com.cloud.utils.db.GenericDao; public interface StoragePoolTagsDao extends GenericDao { - void persist(long poolId, List storagePoolTags); + void persist(long poolId, List storagePoolTags, Boolean isTagARule); + + void persist(List storagePoolTags); List getStoragePoolTags(long poolId); void deleteTags(long poolId); List searchByIds(Long... stIds); StorageTagResponse newStorageTagResponse(StoragePoolTagVO tag); + List findStoragePoolTags(long poolId); + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java index f20e0c411de..c01c66763af 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolTagsDaoImpl.java @@ -21,6 +21,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; @@ -50,7 +53,7 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase storagePoolTags) { + public void persist(long poolId, List storagePoolTags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); @@ -61,13 +64,23 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase 0) { - StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag); + StoragePoolTagVO vo = new StoragePoolTagVO(poolId, tag, isTagARule); persist(vo); } } txn.commit(); } + public void persist(List storagePoolTags) { + Transaction.execute(TransactionLegacy.CLOUD_DB, new TransactionCallbackNoReturn() { + @Override public void doInTransactionWithoutResult(TransactionStatus status) { + for (StoragePoolTagVO storagePoolTagVO : storagePoolTags) { + persist(storagePoolTagVO); + } + } + }); + } + @Override public List getStoragePoolTags(long poolId) { SearchCriteria sc = StoragePoolSearch.create(); @@ -157,4 +170,12 @@ public class StoragePoolTagsDaoImpl extends GenericDaoBase findStoragePoolTags(long poolId) { + SearchCriteria sc = StoragePoolSearch.create(); + sc.setParameters("poolId", poolId); + + return search(sc, null); + } + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 80fddc9bd94..8f77b4ba63e 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -53,7 +53,7 @@ public interface PrimaryDataStoreDao extends GenericDao { */ void updateCapacityIops(long id, long capacityIops); - StoragePoolVO persist(StoragePoolVO pool, Map details, List tags); + StoragePoolVO persist(StoragePoolVO pool, Map details, List tags, Boolean isTagARule); /** * Find pool by name. @@ -77,7 +77,7 @@ public interface PrimaryDataStoreDao extends GenericDao { */ List findPoolsByDetails(long dcId, long podId, Long clusterId, Map details, ScopeType scope); - List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags); + List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout); List findDisabledPoolsByScope(long dcId, Long podId, Long clusterId, ScopeType scope); @@ -112,9 +112,9 @@ public interface PrimaryDataStoreDao extends GenericDao { List listPoolsByCluster(long clusterId); - List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags); + List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule); - List findZoneWideStoragePoolsByTags(long dcId, String[] tags); + List findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule); List findZoneWideStoragePoolsByHypervisor(long dataCenterId, HypervisorType hypervisorType); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index 2fc52b3fb28..af7dbdc0225 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -67,11 +67,11 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase protected final String DetailsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_details ON storage_pool.id = storage_pool_details.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String DetailsSqlSuffix = ") GROUP BY storage_pool_details.pool_id HAVING COUNT(storage_pool_details.name) >= ?"; - private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; + private final String ZoneWideTagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; private final String ZoneWideTagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; // Storage tags are now separate from storage_pool_details, leaving only details on that table - protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; + protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String TagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; private static final String GET_STORAGE_POOLS_OF_VOLUMES_WITHOUT_OR_NOT_HAVING_TAGS = "SELECT s.* " + @@ -278,7 +278,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase @Override @DB - public StoragePoolVO persist(StoragePoolVO pool, Map details, List tags) { + public StoragePoolVO persist(StoragePoolVO pool, Map details, List tags, Boolean isTagARule) { TransactionLegacy txn = TransactionLegacy.currentTxn(); txn.start(); pool = super.persist(pool); @@ -289,7 +289,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } if (CollectionUtils.isNotEmpty(tags)) { - _tagsDao.persist(pool.getId(), tags); + _tagsDao.persist(pool.getId(), tags, isTagARule); } txn.commit(); return pool; @@ -404,10 +404,15 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags) { + public List findPoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule, long ruleExecuteTimeout) { List storagePools = null; if (tags == null || tags.length == 0) { storagePools = listBy(dcId, podId, clusterId, ScopeType.CLUSTER); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } + } else { String sqlValues = getSqlValuesFromStorageTags(tags); storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.CLUSTER, sqlValues, ValueType.TAGS, tags.length); @@ -437,10 +442,14 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags) { + public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule) { List storagePools = null; if (tags == null || tags.length == 0) { storagePools = listBy(dcId, podId, clusterId, ScopeType.HOST); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } } else { String sqlValues = getSqlValuesFromStorageTags(tags); storagePools = findPoolsByDetailsOrTagsInternal(dcId, podId, clusterId, ScopeType.HOST, sqlValues, ValueType.TAGS, tags.length); @@ -483,13 +492,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } @Override - public List findZoneWideStoragePoolsByTags(long dcId, String[] tags) { + public List findZoneWideStoragePoolsByTags(long dcId, String[] tags, boolean validateTagRule) { if (tags == null || tags.length == 0) { QueryBuilder sc = QueryBuilder.create(StoragePoolVO.class); sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId); sc.and(sc.entity().getStatus(), Op.EQ, Status.Up); sc.and(sc.entity().getScope(), Op.EQ, ScopeType.ZONE); - return sc.list(); + + List storagePools = sc.list(); + + if (validateTagRule) { + storagePools = getPoolsWithoutTagRule(storagePools); + } + + return storagePools; } else { String sqlValues = getSqlValuesFromStorageTags(tags); String sql = getSqlPreparedStatement(ZoneWideTagsSqlPrefix, ZoneWideTagsSqlSuffix, sqlValues, null); @@ -497,6 +513,20 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase } } + protected List getPoolsWithoutTagRule(List storagePools) { + List storagePoolsToReturn = new ArrayList<>(); + for (StoragePoolVO storagePool : storagePools) { + + List poolTags = _tagsDao.findStoragePoolTags(storagePool.getId()); + + if (CollectionUtils.isEmpty(poolTags) || !poolTags.get(0).isTagARule()) { + storagePoolsToReturn.add(storagePool); + } + } + + return storagePoolsToReturn; + } + @Override public List searchForStoragePoolTags(long poolId) { return _tagsDao.getStoragePoolTags(poolId); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 61f03be1713..ec6972ccc59 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -224,3 +224,12 @@ CREATE TABLE `cloud`.`oauth_provider` ( `removed` datetime COMMENT 'date removed if not null', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +-- Flexible tags +ALTER TABLE `cloud`.`storage_pool_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; + +ALTER TABLE `cloud`.`storage_pool_tags` MODIFY tag text NOT NULL; + +ALTER TABLE `cloud`.`host_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; + +ALTER TABLE `cloud`.`host_tags` MODIFY tag text NOT NULL; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql new file mode 100644 index 00000000000..5c6d4fd772b --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.host_view.sql @@ -0,0 +1,111 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`host_view`; + +DROP VIEW IF EXISTS `cloud`.`host_view`; + +CREATE VIEW `cloud`.`host_view` AS +SELECT + host.id, + host.uuid, + host.name, + host.status, + host.disconnected, + host.type, + host.private_ip_address, + host.version, + host.hypervisor_type, + host.hypervisor_version, + host.capabilities, + host.last_ping, + host.created, + host.removed, + host.resource_state, + host.mgmt_server_id, + host.cpu_sockets, + host.cpus, + host.speed, + host.ram, + cluster.id cluster_id, + cluster.uuid cluster_uuid, + cluster.name cluster_name, + cluster.cluster_type, + data_center.id data_center_id, + data_center.uuid data_center_uuid, + data_center.name data_center_name, + data_center.networktype data_center_type, + host_pod_ref.id pod_id, + host_pod_ref.uuid pod_uuid, + host_pod_ref.name pod_name, + GROUP_CONCAT(DISTINCT(host_tags.tag)) AS tag, + `host_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + guest_os_category.id guest_os_category_id, + guest_os_category.uuid guest_os_category_uuid, + guest_os_category.name guest_os_category_name, + mem_caps.used_capacity memory_used_capacity, + mem_caps.reserved_capacity memory_reserved_capacity, + cpu_caps.used_capacity cpu_used_capacity, + cpu_caps.reserved_capacity cpu_reserved_capacity, + async_job.id job_id, + async_job.uuid job_uuid, + async_job.job_status job_status, + async_job.account_id job_account_id, + oobm.enabled AS `oobm_enabled`, + oobm.power_state AS `oobm_power_state`, + ha_config.enabled AS `ha_enabled`, + ha_config.ha_state AS `ha_state`, + ha_config.provider AS `ha_provider`, + `last_annotation_view`.`annotation` AS `annotation`, + `last_annotation_view`.`created` AS `last_annotated`, + `user`.`username` AS `username` +FROM + `cloud`.`host` + LEFT JOIN + `cloud`.`cluster` ON host.cluster_id = cluster.id + LEFT JOIN + `cloud`.`data_center` ON host.data_center_id = data_center.id + LEFT JOIN + `cloud`.`host_pod_ref` ON host.pod_id = host_pod_ref.id + LEFT JOIN + `cloud`.`host_details` ON host.id = host_details.host_id + AND host_details.name = 'guest.os.category.id' + LEFT JOIN + `cloud`.`guest_os_category` ON guest_os_category.id = CONVERT ( host_details.value, UNSIGNED ) + LEFT JOIN + `cloud`.`host_tags` ON host_tags.host_id = host.id + LEFT JOIN + `cloud`.`op_host_capacity` mem_caps ON host.id = mem_caps.host_id + AND mem_caps.capacity_type = 0 + LEFT JOIN + `cloud`.`op_host_capacity` cpu_caps ON host.id = cpu_caps.host_id + AND cpu_caps.capacity_type = 1 + LEFT JOIN + `cloud`.`async_job` ON async_job.instance_id = host.id + AND async_job.instance_type = 'Host' + AND async_job.job_status = 0 + LEFT JOIN + `cloud`.`oobm` ON oobm.host_id = host.id + left join + `cloud`.`ha_config` ON ha_config.resource_id=host.id + and ha_config.resource_type='Host' + LEFT JOIN + `cloud`.`last_annotation_view` ON `last_annotation_view`.`entity_uuid` = `host`.`uuid` + LEFT JOIN + `cloud`.`user` ON `user`.`uuid` = `last_annotation_view`.`user_uuid` +GROUP BY + `host`.`id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql new file mode 100644 index 00000000000..e6cc9458208 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.storage_pool_view.sql @@ -0,0 +1,68 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud`.`storage_pool_view`; + +DROP VIEW IF EXISTS `cloud`.`storage_pool_view`; + +CREATE VIEW `cloud`.`storage_pool_view` AS +SELECT + `storage_pool`.`id` AS `id`, + `storage_pool`.`uuid` AS `uuid`, + `storage_pool`.`name` AS `name`, + `storage_pool`.`status` AS `status`, + `storage_pool`.`path` AS `path`, + `storage_pool`.`pool_type` AS `pool_type`, + `storage_pool`.`host_address` AS `host_address`, + `storage_pool`.`created` AS `created`, + `storage_pool`.`removed` AS `removed`, + `storage_pool`.`capacity_bytes` AS `capacity_bytes`, + `storage_pool`.`capacity_iops` AS `capacity_iops`, + `storage_pool`.`scope` AS `scope`, + `storage_pool`.`hypervisor` AS `hypervisor`, + `storage_pool`.`storage_provider_name` AS `storage_provider_name`, + `storage_pool`.`parent` AS `parent`, + `cluster`.`id` AS `cluster_id`, + `cluster`.`uuid` AS `cluster_uuid`, + `cluster`.`name` AS `cluster_name`, + `cluster`.`cluster_type` AS `cluster_type`, + `data_center`.`id` AS `data_center_id`, + `data_center`.`uuid` AS `data_center_uuid`, + `data_center`.`name` AS `data_center_name`, + `data_center`.`networktype` AS `data_center_type`, + `host_pod_ref`.`id` AS `pod_id`, + `host_pod_ref`.`uuid` AS `pod_uuid`, + `host_pod_ref`.`name` AS `pod_name`, + `storage_pool_tags`.`tag` AS `tag`, + `storage_pool_tags`.`is_tag_a_rule` AS `is_tag_a_rule`, + `op_host_capacity`.`used_capacity` AS `disk_used_capacity`, + `op_host_capacity`.`reserved_capacity` AS `disk_reserved_capacity`, + `async_job`.`id` AS `job_id`, + `async_job`.`uuid` AS `job_uuid`, + `async_job`.`job_status` AS `job_status`, + `async_job`.`account_id` AS `job_account_id` +FROM + ((((((`cloud`.`storage_pool` + LEFT JOIN `cloud`.`cluster` ON ((`storage_pool`.`cluster_id` = `cluster`.`id`))) + LEFT JOIN `cloud`.`data_center` ON ((`storage_pool`.`data_center_id` = `data_center`.`id`))) + LEFT JOIN `cloud`.`host_pod_ref` ON ((`storage_pool`.`pod_id` = `host_pod_ref`.`id`))) + LEFT JOIN `cloud`.`storage_pool_tags` ON (((`storage_pool_tags`.`pool_id` = `storage_pool`.`id`)))) + LEFT JOIN `cloud`.`op_host_capacity` ON (((`storage_pool`.`id` = `op_host_capacity`.`host_id`) + AND (`op_host_capacity`.`capacity_type` IN (3 , 9))))) + LEFT JOIN `cloud`.`async_job` ON (((`async_job`.`instance_id` = `storage_pool`.`id`) + AND (`async_job`.`instance_type` = 'StoragePool') + AND (`async_job`.`job_status` = 0)))); diff --git a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java index 9ab010dd6a7..76bc5270b4f 100755 --- a/engine/schema/src/test/java/com/cloud/host/HostVOTest.java +++ b/engine/schema/src/test/java/com/cloud/host/HostVOTest.java @@ -1,61 +1,84 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.host; - -import com.cloud.service.ServiceOfferingVO; -import com.cloud.vm.VirtualMachine; -import java.util.Arrays; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import org.junit.Test; -import org.junit.Before; - -public class HostVOTest { - HostVO host; - ServiceOfferingVO offering; - - @Before - public void setUp() throws Exception { - host = new HostVO(); - offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, - false, "TestSO", false,VirtualMachine.Type.User,false); - } - - @Test - public void testNoSO() { - assertFalse(host.checkHostServiceOfferingTags(null)); - } - - @Test - public void testNoTag() { - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testRightTag() { - host.setHostTags(Arrays.asList("tag1","tag2")); - offering.setHostTag("tag2,tag1"); - assertTrue(host.checkHostServiceOfferingTags(offering)); - } - - @Test - public void testWrongTag() { - host.setHostTags(Arrays.asList("tag1","tag2")); - offering.setHostTag("tag2,tag4"); - assertFalse(host.checkHostServiceOfferingTags(offering)); - } -} +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import com.cloud.service.ServiceOfferingVO; +import com.cloud.vm.VirtualMachine; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import org.junit.Test; +import org.junit.Before; + +public class HostVOTest { + HostVO host; + ServiceOfferingVO offering; + + @Before + public void setUp() throws Exception { + host = new HostVO(); + offering = new ServiceOfferingVO("TestSO", 0, 0, 0, 0, 0, + false, "TestSO", false,VirtualMachine.Type.User,false); + } + + @Test + public void testNoSO() { + assertFalse(host.checkHostServiceOfferingTags(null)); + } + + @Test + public void testNoTag() { + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testRightTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag1"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void testWrongTag() { + host.setHostTags(Arrays.asList("tag1","tag2"), false); + offering.setHostTag("tag2,tag4"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatMatches() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("A"); + assertTrue(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithServiceTagThatDoesNotMatch() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag("B"); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } + + @Test + public void checkHostServiceOfferingTagsTestRuleTagWithNullServiceTag() { + host.setHostTags(List.of("tags[0] == 'A'"), true); + offering.setHostTag(null); + assertFalse(host.checkHostServiceOfferingTags(offering)); + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index f2b0f17232b..89a7b577ae7 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -28,6 +28,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.api.query.dao.StoragePoolJoinDao; import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.ScopeType; import com.cloud.storage.StoragePoolStatus; @@ -85,6 +86,9 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement */ private SecureRandom secureRandom = new SecureRandom(); + @Inject + protected StoragePoolJoinDao storagePoolJoinDao; + @Override public boolean configure(String name, Map params) throws ConfigurationException { super.configure(name, params); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java index fe49504fda1..9c0f84ab14a 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.storage.VolumeApiServiceImpl; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -78,11 +79,12 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat logDisabledStoragePools(dcId, podId, clusterId, ScopeType.CLUSTER); } - List pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags()); + List pools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, dskCh.getTags(), true, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value()); + pools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(dcId, podId, clusterId, ScopeType.CLUSTER, List.of(dskCh.getTags()))); s_logger.debug(String.format("Found pools [%s] that match with tags [%s].", pools, Arrays.toString(dskCh.getTags()))); // add remaining pools in cluster, that did not match tags, to avoid set - List allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null); + List allPools = storagePoolDao.findPoolsByTags(dcId, podId, clusterId, null, false, 0); allPools.removeAll(pools); for (StoragePoolVO pool : allPools) { s_logger.trace(String.format("Adding pool [%s] to the 'avoid' set since it did not match any tags.", pool)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java index 774c2229a09..4ec15b9e43f 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/LocalStoragePoolAllocator.java @@ -101,7 +101,8 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { return null; } List availablePools = - storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags()); + storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), dskCh.getTags(), true); + availablePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), ScopeType.HOST, List.of(dskCh.getTags()))); for (StoragePoolVO pool : availablePools) { if (suitablePools.size() == returnUpTo) { break; @@ -117,7 +118,7 @@ public class LocalStoragePoolAllocator extends AbstractStoragePoolAllocator { } // add remaining pools in cluster to the 'avoid' set which did not match tags - List allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null); + List allPools = storagePoolDao.findLocalStoragePoolsByTags(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), null, false); allPools.removeAll(availablePools); for (StoragePoolVO pool : allPools) { avoid.addPool(pool.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java index 1b3835560df..ba130b4e2e5 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java @@ -63,9 +63,9 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { } List suitablePools = new ArrayList<>(); - - List storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags()); - if (storagePools == null) { + List storagePools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), dskCh.getTags(), true); + storagePools.addAll(storagePoolJoinDao.findStoragePoolByScopeAndRuleTags(plan.getDataCenterId(), null, null, ScopeType.ZONE, List.of(dskCh.getTags()))); + if (storagePools.isEmpty()) { LOGGER.debug(String.format("Could not find any zone wide storage pool that matched with any of the following tags [%s].", Arrays.toString(dskCh.getTags()))); storagePools = new ArrayList<>(); } @@ -82,7 +82,7 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { storagePools.addAll(anyHypervisorStoragePools); // add remaining pools in zone, that did not match tags, to avoid set - List allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null); + List allPools = storagePoolDao.findZoneWideStoragePoolsByTags(plan.getDataCenterId(), null, false); allPools.removeAll(storagePools); for (StoragePoolVO pool : allPools) { avoid.addPool(pool.getId()); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java index c3379ad316b..fbb4a6e1618 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/volume/datastore/PrimaryDataStoreHelper.java @@ -136,7 +136,7 @@ public class PrimaryDataStoreHelper { storageTags.add(tag); } } - dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags); + dataStoreVO = dataStoreDao.persist(dataStoreVO, details, storageTags, params.isTagARule()); return dataStoreMgr.getDataStore(dataStoreVO.getId(), DataStoreRole.Primary); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java index 337131038e8..fef3e4376dc 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Host.java @@ -22,6 +22,8 @@ import java.util.List; public class Host extends GenericPresetVariable { private List tags; + private Boolean isTagARule; + public List getTags() { return tags; } @@ -31,4 +33,12 @@ public class Host extends GenericPresetVariable { fieldNamesToIncludeInToString.add("tags"); } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + fieldNamesToIncludeInToString.add("isTagARule"); + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 6ce8bd889a6..9723d3e5899 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -25,9 +25,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import com.cloud.host.HostTagVO; import javax.inject.Inject; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -351,7 +353,17 @@ public class PresetVariableHelper { Host host = new Host(); host.setId(hostVo.getUuid()); host.setName(hostVo.getName()); - host.setTags(hostTagsDao.getHostTags(hostId)); + List hostTagVOList = hostTagsDao.getHostTags(hostId); + List hostTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(hostTagVOList)) { + isTagARule = hostTagVOList.get(0).getIsTagARule(); + if (!isTagARule) { + hostTags = hostTagVOList.parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); + } + } + host.setTags(hostTags); + host.setIsTagARule(isTagARule); return host; } @@ -508,7 +520,17 @@ public class PresetVariableHelper { storage.setId(storagePoolVo.getUuid()); storage.setName(storagePoolVo.getName()); storage.setScope(storagePoolVo.getScope()); - storage.setTags(storagePoolTagsDao.getStoragePoolTags(storageId)); + List storagePoolTagVOList = storagePoolTagsDao.findStoragePoolTags(storageId); + List storageTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) { + isTagARule = storagePoolTagVOList.get(0).isTagARule(); + if (!isTagARule) { + storageTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + } + } + storage.setTags(storageTags); + storage.setIsTagARule(isTagARule); return storage; } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java index 3533c5d45c1..6be1dfb025a 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Storage.java @@ -23,6 +23,8 @@ import com.cloud.storage.ScopeType; public class Storage extends GenericPresetVariable { private List tags; + + private Boolean isTagARule; private ScopeType scope; public List getTags() { @@ -34,6 +36,15 @@ public class Storage extends GenericPresetVariable { fieldNamesToIncludeInToString.add("tags"); } + public Boolean getIsTagARule() { + return isTagARule; + } + + public void setIsTagARule(Boolean isTagARule) { + this.isTagARule = isTagARule; + fieldNamesToIncludeInToString.add("isTagARule"); + } + public ScopeType getScope() { return scope; } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index ae15e573fa8..b973d1145c3 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -27,7 +27,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.StoragePoolTagVO; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; @@ -241,6 +243,14 @@ public class PresetVariableHelperTest { return host; } + private List getHostTagsForTests() { + return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); + } + + private List getHostRuleTagsForTests() { + return List.of(new HostTagVO(1, "tagrule", true)); + } + private Storage getStorageForTests() { Storage storage = new Storage(); storage.setId("storage_id"); @@ -250,6 +260,14 @@ public class PresetVariableHelperTest { return storage; } + private List getStorageTagsForTests() { + return Arrays.asList(new StoragePoolTagVO(1, "tag1", false), new StoragePoolTagVO(1, "tag2", false)); + } + + private List getStorageRuleTagsForTests() { + return List.of(new StoragePoolTagVO(1, "tagrule", true)); + } + private Set> getQuotaTypesForTests(Integer... typesToRemove) { Map quotaTypesMap = new LinkedHashMap<>(QuotaTypes.listQuotaTypes()); @@ -539,15 +557,16 @@ public class PresetVariableHelperTest { @Test public void setPresetVariableHostInValueIfUsageTypeIsRunningVmTestQuotaTypeIsRunningVmSetHost() { Value result = new Value(); - Host expected = getHostForTests(); + Host expectedHost = getHostForTests(); + List expectedHostTags = getHostTagsForTests(); - Mockito.doReturn(expected).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong()); + Mockito.doReturn(expectedHost).when(presetVariableHelperSpy).getPresetVariableValueHost(Mockito.anyLong()); presetVariableHelperSpy.setPresetVariableHostInValueIfUsageTypeIsRunningVm(result, UsageTypes.RUNNING_VM, vmInstanceVoMock); Assert.assertNotNull(result.getHost()); - assertPresetVariableIdAndName(expected, result.getHost()); - Assert.assertEquals(expected.getTags(), result.getHost().getTags()); + assertPresetVariableIdAndName(expectedHost, result.getHost()); + Assert.assertEquals(expectedHost.getTags(), result.getHost().getTags()); validateFieldNamesToIncludeInToString(Arrays.asList("host"), result); } @@ -555,18 +574,39 @@ public class PresetVariableHelperTest { public void getPresetVariableValueHostTestSetFieldsAndReturnObject() { Host expected = getHostForTests(); HostVO hostVoMock = Mockito.mock(HostVO.class); + List hostTagVOListMock = getHostTagsForTests(); Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); mockMethodValidateIfObjectIsNull(); Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(hostVoMock).getName(); - Mockito.doReturn(expected.getTags()).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); + Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l); assertPresetVariableIdAndName(expected, result); Assert.assertEquals(expected.getTags(), result.getTags()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "tags"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result); + } + + @Test + public void getPresetVariableValueHostTestSetFieldsWithRuleTagAndReturnObject() { + Host expected = getHostForTests(); + HostVO hostVoMock = Mockito.mock(HostVO.class); + List hostTagVOListMock = getHostRuleTagsForTests(); + + Mockito.doReturn(hostVoMock).when(hostDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + Mockito.doReturn(expected.getId()).when(hostVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(hostVoMock).getName(); + Mockito.doReturn(hostTagVOListMock).when(hostTagsDaoMock).getHostTags(Mockito.anyLong()); + + Host result = presetVariableHelperSpy.getPresetVariableValueHost(1l); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(new ArrayList<>(), result.getTags()); + Assert.assertTrue(result.getIsTagARule()); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "tags"), result); } @Test @@ -755,13 +795,15 @@ public class PresetVariableHelperTest { Storage expected = getStorageForTests(); Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt()); + List storageTagVOListMock = getStorageTagsForTests(); + StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class); Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid(); Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName(); Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope(); - Mockito.doReturn(expected.getTags()).when(storagePoolTagsDaoMock).getStoragePoolTags(Mockito.anyLong()); + Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong()); Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2); @@ -769,7 +811,32 @@ public class PresetVariableHelperTest { Assert.assertEquals(expected.getScope(), result.getScope()); Assert.assertEquals(expected.getTags(), result.getTags()); - validateFieldNamesToIncludeInToString(Arrays.asList("id", "name", "scope", "tags"), result); + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "scope", "tags"), result); + } + + @Test + public void getPresetVariableValueStorageTestGetSecondaryStorageForSnapshotReturnsNullWithRuleTag() { + Storage expected = getStorageForTests(); + Mockito.doReturn(null).when(presetVariableHelperSpy).getSecondaryStorageForSnapshot(Mockito.anyLong(), Mockito.anyInt()); + + List storageTagVOListMock = getStorageRuleTagsForTests(); + + StoragePoolVO storagePoolVoMock = Mockito.mock(StoragePoolVO.class); + Mockito.doReturn(storagePoolVoMock).when(primaryStorageDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + Mockito.doReturn(expected.getId()).when(storagePoolVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(storagePoolVoMock).getName(); + Mockito.doReturn(expected.getScope()).when(storagePoolVoMock).getScope(); + Mockito.doReturn(storageTagVOListMock).when(storagePoolTagsDaoMock).findStoragePoolTags(Mockito.anyLong()); + + Storage result = presetVariableHelperSpy.getPresetVariableValueStorage(1l, 2); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getScope(), result.getScope()); + Assert.assertEquals(new ArrayList<>(), result.getTags()); + Assert.assertTrue(result.getIsTagARule()); + + validateFieldNamesToIncludeInToString(Arrays.asList("id", "isTagARule", "name", "scope", "tags"), result); } @Test diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java index 8a46d10a7b5..70920df5eb5 100644 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java @@ -22,7 +22,9 @@ import java.util.List; import javax.inject.Inject; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.ListUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -72,7 +74,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { } String hostTag = offering.getHostTag(); if (hostTag != null) { - s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId + " having host tag:" + hostTag); + s_logger.debug(String.format("Looking for hosts in dc [%s], pod [%s], cluster [%s] and complying with host tag [%s].", dcId, podId, clusterId, hostTag)); } else { s_logger.debug("Looking for hosts in dc: " + dcId + " pod:" + podId + " cluster:" + clusterId); } @@ -82,7 +84,7 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { if (hostTag != null) { hostsCopy.retainAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag)); } else { - hostsCopy.retainAll(_resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId)); + hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); } } else { // list all computing hosts, regardless of whether they support routing...it's random after all @@ -90,9 +92,16 @@ public class RandomAllocator extends AdapterBase implements HostAllocator { if (hostTag != null) { hostsCopy = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTag); } else { - hostsCopy = _resourceMgr.listAllUpAndEnabledHosts(type, clusterId, podId, dcId); + hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); } } + hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTag)); + + if (hostsCopy.isEmpty()) { + s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTag)); + throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + } + s_logger.debug("Random Allocator found " + hostsCopy.size() + " hosts"); if (hostsCopy.size() == 0) { return suitableHosts; diff --git a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java index ee48252534f..3b111da5961 100644 --- a/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java +++ b/plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java @@ -29,6 +29,7 @@ import com.cloud.configuration.dao.ResourceCountDao; import com.cloud.dc.DedicatedResourceVO; import com.cloud.dc.dao.DedicatedResourceDao; import com.cloud.host.HostStats; +import com.cloud.host.HostTagVO; import com.cloud.user.Account; import com.cloud.user.dao.AccountDao; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; @@ -234,7 +235,9 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp } private String markTagMaps(HostVO host, Map totalHosts, Map upHosts, Map downHosts) { - List hostTags = _hostTagsDao.getHostTags(host.getId()); + List hostTagVOS = _hostTagsDao.getHostTags(host.getId()); + List hostTags = new ArrayList<>(); + hostTagVOS.forEach(hostTagVO -> hostTags.add(hostTagVO.getTag())); markTags(hostTags,totalHosts); if (host.getStatus() == Status.Up && !host.isInMaintenanceStates()) { markTags(hostTags, upHosts); @@ -277,10 +280,12 @@ public class PrometheusExporterImpl extends ManagerBase implements PrometheusExp metricsList.add(new ItemHostMemory(zoneName, zoneUuid, null, null, null, null, ALLOCATED, allocatedCapacityByTag.third(), 0, tag)); }); - List allHostTags = hostDao.listAll().stream() + List allHostTagVOS = hostDao.listAll().stream() .flatMap( h -> _hostTagsDao.getHostTags(h.getId()).stream()) .distinct() .collect(Collectors.toList()); + List allHostTags = new ArrayList<>(); + allHostTagVOS.forEach(hostTagVO -> allHostTags.add(hostTagVO.getTag())); for (final State state : State.values()) { for (final String hostTag : allHostTags) { diff --git a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java index d6a67b447c5..0798f9f2cd2 100644 --- a/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/cloudbyte/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ElastistorPrimaryDataStoreLifeCycle.java @@ -107,6 +107,7 @@ public class ElastistorPrimaryDataStoreLifeCycle implements PrimaryDataStoreLife Long capacityBytes = (Long) dsInfos.get("capacityBytes"); Long capacityIops = (Long) dsInfos.get("capacityIops"); String tags = (String) dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); boolean managed = (Boolean) dsInfos.get("managed"); Map details = (Map) dsInfos.get("details"); String domainName = details.get("domainname"); @@ -196,6 +197,7 @@ public class ElastistorPrimaryDataStoreLifeCycle implements PrimaryDataStoreLife parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(HypervisorType.Any); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); parameters.setClusterId(clusterId); diff --git a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java index 0906e645b66..6fd42009125 100644 --- a/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/datera/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/DateraPrimaryDataStoreLifeCycle.java @@ -100,6 +100,7 @@ public class DateraPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCycl Long capacityBytes = (Long) dsInfos.get("capacityBytes"); Long capacityIops = (Long) dsInfos.get("capacityIops"); String tags = (String) dsInfos.get("tags"); + boolean isTagARule = (Boolean)dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map) dsInfos.get("details"); String domainName = details.get("domainname"); @@ -181,6 +182,7 @@ public class DateraPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCycl parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(HypervisorType.Any); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); String managementVip = DateraUtil.getManagementVip(url); diff --git a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 6e178c119e3..685565d73b0 100644 --- a/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -138,6 +138,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore Map details = (Map)dsInfos.get("details"); parameters.setTags(tags); + parameters.setIsTagARule((Boolean)dsInfos.get("isTagARule")); parameters.setDetails(details); String scheme = dsInfos.get("scheme").toString(); diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java index a7d0c7fc0a3..efc69438e75 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/LinstorPrimaryDataStoreLifeCycleImpl.java @@ -91,6 +91,7 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif String providerName = (String) dsInfos.get("providerName"); Long capacityIops = (Long) dsInfos.get("capacityIops"); String tags = (String) dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map) dsInfos.get("details"); @@ -168,6 +169,7 @@ public class LinstorPrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLif parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(HypervisorType.KVM); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); parameters.setUserInfo(resourceGroup); diff --git a/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java index 32735664a25..507189edc14 100644 --- a/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/nexenta/src/main/java/org/apache/cloudstack/storage/datastore/lifecylce/NexentaPrimaryDataStoreLifeCycle.java @@ -69,6 +69,7 @@ public class NexentaPrimaryDataStoreLifeCycle Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); Map details = (Map) dsInfos.get("details"); NexentaUtil.NexentaPluginParameters params = NexentaUtil.parseNexentaPluginUrl(url); DataCenterVO zone = zoneDao.findById(zoneId); @@ -98,6 +99,7 @@ public class NexentaPrimaryDataStoreLifeCycle parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(Hypervisor.HypervisorType.Any); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); details.put(NexentaUtil.NMS_URL, params.getNmsUrl().toString()); diff --git a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java index 65831e4ec3d..17150699923 100644 --- a/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/ScaleIOPrimaryDataStoreLifeCycle.java @@ -139,6 +139,7 @@ public class ScaleIOPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCyc Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); Map details = (Map) dsInfos.get("details"); if (zoneId == null) { @@ -224,6 +225,7 @@ public class ScaleIOPrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCyc parameters.setHypervisorType(Hypervisor.HypervisorType.KVM); parameters.setUuid(UUID.randomUUID().toString()); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); StoragePoolStatistics poolStatistics = scaleIOPool.getStatistics(); if (poolStatistics != null) { diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java index 38c8c240094..7a2767c32e6 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java @@ -91,6 +91,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map)dsInfos.get("details"); @@ -142,6 +143,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC } parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); String managementVip = SolidFireUtil.getManagementVip(url); diff --git a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java index 9cc746d4ee8..557cc3f60f6 100644 --- a/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/solidfire/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SolidFireSharedPrimaryDataStoreLifeCycle.java @@ -104,6 +104,7 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get(CAPACITY_IOPS); String tags = (String)dsInfos.get("tags"); + Boolean isTagARule = (Boolean) dsInfos.get("isTagARule"); @SuppressWarnings("unchecked") Map details = (Map)dsInfos.get("details"); @@ -152,6 +153,7 @@ public class SolidFireSharedPrimaryDataStoreLifeCycle implements PrimaryDataStor parameters.setCapacityIops(capacityIops); parameters.setHypervisorType(hypervisorType); parameters.setTags(tags); + parameters.setIsTagARule(isTagARule); parameters.setDetails(details); String managementVip = SolidFireUtil.getManagementVip(url); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index df6ea74881e..862e8acc9e0 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -25,6 +25,7 @@ import java.util.Map; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -182,7 +183,6 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { if (hasSvcOfferingTag && hasTemplateTag) { hostsMatchingOfferingTag.retainAll(hostsMatchingTemplateTag); - clusterHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, hostTagOnTemplate); if (s_logger.isDebugEnabled()) { s_logger.debug("Found " + hostsMatchingOfferingTag.size() + " Hosts satisfying both tags, host ids are:" + hostsMatchingOfferingTag); } @@ -202,6 +202,13 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { clusterHosts.retainAll(hostsMatchingUefiTag); } + clusterHosts.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); + + + if (clusterHosts.isEmpty()) { + s_logger.error(String.format("No suitable host found for vm [%s] with tags [%s].", vmProfile, hostTagOnOffering)); + throw new CloudRuntimeException(String.format("No suitable host found for vm [%s].", vmProfile)); + } // add all hosts that we are not considering to the avoid list List allhostsInCluster = _hostDao.listAllUpAndEnabledNonHAHosts(type, clusterId, podId, dcId, null); allhostsInCluster.removeAll(clusterHosts); @@ -267,6 +274,8 @@ public class FirstFitAllocator extends AdapterBase implements HostAllocator { } } + hostsCopy.addAll(_hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(hostTagOnOffering)); + if (!hostsCopy.isEmpty()) { suitableHosts = allocateTo(plan, offering, template, avoid, hostsCopy, returnUpTo, considerReservedCapacity, account); } diff --git a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java index bf3ded1fef0..da81f42b41d 100644 --- a/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/HostJoinDaoImpl.java @@ -205,6 +205,7 @@ public class HostJoinDaoImpl extends GenericDaoBase implements String hostTags = host.getTag(); hostResponse.setHostTags(hostTags); + hostResponse.setIsTagARule(host.getIsTagARule()); hostResponse.setHaHost(containsHostHATag(hostTags)); hostResponse.setHypervisorVersion(host.getHypervisorVersion()); diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java index 87659210ad7..9028f3418b3 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDao.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.response.StoragePoolResponse; import com.cloud.api.query.vo.StoragePoolJoinVO; import com.cloud.storage.StoragePool; import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; public interface StoragePoolJoinDao extends GenericDao { @@ -44,4 +45,6 @@ public interface StoragePoolJoinDao extends GenericDao Pair, Integer> searchAndCount(Long storagePoolId, String storagePoolName, Long zoneId, String path, Long podId, Long clusterId, String address, ScopeType scopeType, StoragePoolStatus status, String keyword, Filter searchFilter); + List findStoragePoolByScopeAndRuleTags(Long datacenterId, Long podId, Long clusterId, ScopeType scopeType, List tags); + } diff --git a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java index de469d21a11..e75e86108c7 100644 --- a/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/StoragePoolJoinDaoImpl.java @@ -25,6 +25,7 @@ import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.StorageStats; +import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.user.AccountManager; import com.cloud.utils.Pair; import com.cloud.utils.StringUtils; @@ -44,6 +45,7 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -71,10 +73,13 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase spSearch; private final SearchBuilder spIdSearch; + private final SearchBuilder findByDatacenterAndScopeSb; + protected StoragePoolJoinDaoImpl() { spSearch = createSearchBuilder(); @@ -85,6 +90,15 @@ public class StoragePoolJoinDaoImpl extends GenericDaoBase findStoragePoolByScopeAndRuleTags(Long datacenterId, Long podId, Long clusterId, ScopeType scopeType, List tags) { + SearchCriteria sc = findByDatacenterAndScopeSb.create(); + if (datacenterId != null) { + sc.setParameters("zoneId", datacenterId); + } + if (clusterId != null) { + sc.setParameters("clusterId", clusterId); + } + if (podId != null) { + sc.setParameters("podId", podId); + } + + sc.setParameters("scope", scopeType); + sc.setParameters("status", "Up"); + sc.setParameters("is_tag_a_rule", true); + List storagePools = search(sc, null, false, false); + + List filteredPools = new ArrayList<>(); + + StringBuilder injectableTagsBuilder = new StringBuilder(); + for (String tag : tags) { + injectableTagsBuilder.append(tag).append(","); + } + if (!tags.isEmpty()) { + injectableTagsBuilder.deleteCharAt(injectableTagsBuilder.length() - 1); + } + String injectableTag = injectableTagsBuilder.toString(); + + for (StoragePoolJoinVO storagePoolJoinVO : storagePools) { + if (TagAsRuleHelper.interpretTagAsRule(storagePoolJoinVO.getTag(), injectableTag, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value())) { + StoragePoolVO storagePoolVO = storagePoolDao.findById(storagePoolJoinVO.getId()); + if (storagePoolVO != null) { + filteredPools.add(storagePoolVO); + } else { + s_logger.warn(String.format("Unable to find Storage Pool [%s] in the DB.", storagePoolJoinVO.getUuid())); + } + } + } + return filteredPools; + } + } diff --git a/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java index bb3b6935389..78a45429463 100644 --- a/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/HostJoinVO.java @@ -172,6 +172,9 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private Boolean isTagARule; + @Column(name = "memory_used_capacity") private long memUsedCapacity; @@ -388,6 +391,10 @@ public class HostJoinVO extends BaseViewVO implements InternalIdentity, Identity return tag; } + public Boolean getIsTagARule() { + return isTagARule; + } + public String getAnnotation() { return annotation; } diff --git a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java index 1831aaafac9..5eb04d2e00d 100644 --- a/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java +++ b/server/src/main/java/com/cloud/api/query/vo/StoragePoolJoinVO.java @@ -110,6 +110,9 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I @Column(name = "tag") private String tag; + @Column(name = "is_tag_a_rule") + private boolean isTagARule; + @Column(name = "disk_used_capacity") private long usedCapacity; @@ -243,6 +246,10 @@ public class StoragePoolJoinVO extends BaseViewVO implements InternalIdentity, I return tag; } + public boolean getIsTagARule() { + return isTagARule; + } + public long getUsedCapacity() { return usedCapacity; } diff --git a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java index acf57a788a0..502fccfde9e 100644 --- a/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java +++ b/server/src/main/java/com/cloud/configuration/ConfigurationManagerImpl.java @@ -48,6 +48,10 @@ import javax.naming.ConfigurationException; import com.cloud.hypervisor.HypervisorGuru; import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.host.HostTagVO; +import com.cloud.storage.StoragePoolTagVO; +import com.cloud.storage.VolumeApiServiceImpl; +import com.googlecode.ipv6.IPv6Address; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupService; @@ -128,6 +132,7 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -293,7 +298,6 @@ import com.google.common.base.Enums; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; -import com.googlecode.ipv6.IPv6Address; import com.googlecode.ipv6.IPv6Network; public class ConfigurationManagerImpl extends ManagerBase implements ConfigurationManager, ConfigurationService, Configurable { @@ -4080,17 +4084,23 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (CollectionUtils.isNotEmpty(pools)) { List listOfTags = Arrays.asList(tags.split(",")); for (StoragePoolVO storagePoolVO : pools) { - List tagsOnPool = storagePoolTagDao.getStoragePoolTags(storagePoolVO.getId()); - if (CollectionUtils.isEmpty(tagsOnPool) || !tagsOnPool.containsAll(listOfTags)) { - DiskOfferingVO offeringToRetrieveInfo = _diskOfferingDao.findById(diskOffering.getId()); - List volumes = _volumeDao.findByDiskOfferingId(diskOffering.getId()); - String listOfVolumesNamesAndUuid = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(volumes, "name", "uuid"); - String diskOfferingInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(offeringToRetrieveInfo, "name", "uuid"); - String poolInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(storagePoolVO, "name", "uuid"); - throw new InvalidParameterValueException(String.format("There are active volumes using the disk offering %s, and the pool %s doesn't have the new tags. " + - "The following volumes are using the mentioned disk offering %s. Please first add the new tags to the mentioned storage pools before adding them" + - " to the disk offering.", diskOfferingInfo, poolInfo, listOfVolumesNamesAndUuid)); + List tagsOnPool = storagePoolTagDao.findStoragePoolTags(storagePoolVO.getId()); + List tagsAsString = tagsOnPool.stream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + + if ((CollectionUtils.isNotEmpty(tagsAsString) && tagsAsString.containsAll(listOfTags)) || + (tagsOnPool.size() == 1 && tagsOnPool.get(0).isTagARule() && + TagAsRuleHelper.interpretTagAsRule(tagsOnPool.get(0).getTag(), tags, VolumeApiServiceImpl.storageTagRuleExecutionTimeout.value()))) { + continue; } + + DiskOfferingVO offeringToRetrieveInfo = _diskOfferingDao.findById(diskOffering.getId()); + List volumes = _volumeDao.findByDiskOfferingId(diskOffering.getId()); + String listOfVolumesNamesAndUuid = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(volumes, "name", "uuid"); + String diskOfferingInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(offeringToRetrieveInfo, "name", "uuid"); + String poolInfo = ReflectionToStringBuilderUtils.reflectOnlySelectedFields(storagePoolVO, "name", "uuid"); + throw new InvalidParameterValueException(String.format("There are active volumes using the disk offering %s, and the pool %s doesn't have the new tags. " + + "The following volumes are using the mentioned disk offering %s. Please first add the new tags to the mentioned storage pools before adding them" + + " to the disk offering.", diskOfferingInfo, poolInfo, listOfVolumesNamesAndUuid)); } } diskOffering.setTags(tags); @@ -4117,10 +4127,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (CollectionUtils.isNotEmpty(hosts)) { List listOfHostTags = Arrays.asList(hostTags.split(",")); for (HostVO host : hosts) { - List tagsOnHost = hostTagDao.getHostTags(host.getId()); - if (CollectionUtils.isEmpty(tagsOnHost) || !tagsOnHost.containsAll(listOfHostTags)) { - throw new InvalidParameterValueException(String.format("There are active VMs using offering [%s], and the hosts [%s] don't have the new tags", offering.getId(), hosts)); + List tagsOnHost = hostTagDao.getHostTags(host.getId()); + List tagsAsString = tagsOnHost.stream().map(HostTagVO::getTag).collect(Collectors.toList()); + + if ((CollectionUtils.isNotEmpty(tagsAsString) && tagsAsString.containsAll(listOfHostTags)) || + (tagsOnHost.size() == 1 && tagsOnHost.get(0).getIsTagARule() && + TagAsRuleHelper.interpretTagAsRule(tagsOnHost.get(0).getTag(), hostTags, HostTagsDao.hostTagRuleExecutionTimeout.value()))) { + continue; } + + throw new InvalidParameterValueException(String.format("There are active VMs using offering [%s], and the hosts [%s] don't have the new tags", + offering.getId(), hosts)); } } offering.setHostTag(hostTags); diff --git a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java index 9cb7f4e8aaf..cb22e81f366 100644 --- a/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java +++ b/server/src/main/java/com/cloud/deploy/DeploymentPlanningManagerImpl.java @@ -1361,7 +1361,7 @@ StateListener, Configurable { if (vmRequiresSharedStorage) { // check shared pools - List allPoolsInCluster = _storagePoolDao.findPoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null); + List allPoolsInCluster = _storagePoolDao.findPoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null, false, 0); for (StoragePoolVO pool : allPoolsInCluster) { if (!allocatorAvoidOutput.shouldAvoid(pool)) { // there's some pool in the cluster that is not yet in avoid set @@ -1374,7 +1374,7 @@ StateListener, Configurable { if (vmRequiresLocalStorege) { // check local pools List allLocalPoolsInCluster = - _storagePoolDao.findLocalStoragePoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null); + _storagePoolDao.findLocalStoragePoolsByTags(clusterVO.getDataCenterId(), clusterVO.getPodId(), clusterVO.getId(), null, false); for (StoragePoolVO pool : allLocalPoolsInCluster) { if (!allocatorAvoidOutput.shouldAvoid(pool)) { // there's some pool in the cluster that is not yet diff --git a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java index 4e838510010..c2969ecce50 100644 --- a/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java +++ b/server/src/main/java/com/cloud/deploy/FirstFitPlanner.java @@ -27,6 +27,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.capacity.CapacityVO; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; @@ -521,6 +522,12 @@ public class FirstFitPlanner extends AdapterBase implements DeploymentClusterPla private void removeClustersWithoutMatchingTag(List clusterListForVmAllocation, String hostTagOnOffering) { List matchingClusters = hostDao.listClustersByHostTag(hostTagOnOffering); + matchingClusters.addAll(hostDao.findClustersThatMatchHostTagRule(hostTagOnOffering)); + + if (matchingClusters.isEmpty()) { + s_logger.error(String.format("No suitable host found for the following compute offering tags [%s].", hostTagOnOffering)); + throw new CloudRuntimeException("No suitable host found."); + } clusterListForVmAllocation.retainAll(matchingClusters); diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index d1697646c39..922df25a726 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -32,11 +32,13 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Random; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.alert.AlertManager; +import com.cloud.host.HostTagVO; import com.cloud.exception.StorageConflictException; import com.cloud.exception.StorageUnavailableException; import com.cloud.storage.Volume; @@ -856,7 +858,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (s_logger.isTraceEnabled()) { s_logger.trace("Adding Host Tags for KVM host, tags: :" + hostTags); } - _hostTagsDao.persist(host.getId(), hostTags); + _hostTagsDao.persist(host.getId(), hostTags, false); } hosts.add(host); @@ -1909,7 +1911,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } } - private void updateHostTags(HostVO host, Long hostId, List hostTags) { + private void updateHostTags(HostVO host, Long hostId, List hostTags, Boolean isTagARule) { List activeVMs = _vmDao.listByHostId(hostId); s_logger.warn(String.format("The following active VMs [%s] are using the host [%s]. " + "Updating the host tags will not affect them.", activeVMs, host)); @@ -1917,17 +1919,17 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, if (s_logger.isDebugEnabled()) { s_logger.debug("Updating Host Tags to :" + hostTags); } - _hostTagsDao.persist(hostId, new ArrayList<>(new HashSet<>(hostTags))); + _hostTagsDao.persist(hostId, new ArrayList<>(new HashSet<>(hostTags)), isTagARule); } @Override public Host updateHost(final UpdateHostCmd cmd) throws NoTransitionException { return updateHost(cmd.getId(), cmd.getName(), cmd.getOsCategoryId(), - cmd.getAllocationState(), cmd.getUrl(), cmd.getHostTags(), cmd.getAnnotation(), false); + cmd.getAllocationState(), cmd.getUrl(), cmd.getHostTags(), cmd.getIsTagARule(), cmd.getAnnotation(), false); } private Host updateHost(Long hostId, String name, Long guestOSCategoryId, String allocationState, - String url, List hostTags, String annotation, boolean isUpdateFromHostHealthCheck) throws NoTransitionException { + String url, List hostTags, Boolean isTagARule, String annotation, boolean isUpdateFromHostHealthCheck) throws NoTransitionException { // Verify that the host exists final HostVO host = _hostDao.findById(hostId); if (host == null) { @@ -1948,7 +1950,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, } if (hostTags != null) { - updateHostTags(host, hostId, hostTags); + updateHostTags(host, hostId, hostTags, isTagARule); } if (url != null) { @@ -2007,7 +2009,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public Host autoUpdateHostAllocationState(Long hostId, ResourceState.Event resourceEvent) throws NoTransitionException { - return updateHost(hostId, null, null, resourceEvent.toString(), null, null, null, true); + return updateHost(hostId, null, null, resourceEvent.toString(), null, null, null, null, true); } @Override @@ -2339,7 +2341,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, final List implicitHostTags = ssCmd.getHostTags(); if (!implicitHostTags.isEmpty()) { if (hostTags == null) { - hostTags = _hostTagsDao.getHostTags(host.getId()); + hostTags = _hostTagsDao.getHostTags(host.getId()).parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); } if (hostTags != null) { implicitHostTags.removeAll(hostTags); @@ -2367,7 +2369,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, host.setManagementServerId(_nodeId); host.setStorageUrl(startup.getIqn()); host.setLastPinged(System.currentTimeMillis() >> 10); - host.setHostTags(hostTags); + host.setHostTags(hostTags, false); host.setDetails(details); if (startup.getStorageIpAddressDeux() != null) { host.setStorageIpAddressDeux(startup.getStorageIpAddressDeux()); @@ -3351,7 +3353,7 @@ public class ResourceManagerImpl extends ManagerBase implements ResourceManager, @Override public String getHostTags(final long hostId) { - final List hostTags = _hostTagsDao.getHostTags(hostId); + final List hostTags = _hostTagsDao.getHostTags(hostId).parallelStream().map(HostTagVO::getTag).collect(Collectors.toList()); if (hostTags == null) { return null; } else { diff --git a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java index d881dee137a..25b2ad53bf2 100644 --- a/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/RollingMaintenanceManagerImpl.java @@ -54,6 +54,7 @@ import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; import com.cloud.host.Host; +import com.cloud.host.HostTagVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; @@ -615,7 +616,7 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin if (CollectionUtils.isEmpty(vmsRunning)) { return new Pair<>(true, "OK"); } - List hostTags = hostTagsDao.getHostTags(host.getId()); + List hostTags = hostTagsDao.getHostTags(host.getId()); int successfullyCheckedVmMigrations = 0; for (VMInstanceVO runningVM : vmsRunning) { @@ -668,14 +669,14 @@ public class RollingMaintenanceManagerImpl extends ManagerBase implements Rollin /** * Check hosts tags */ - private boolean checkHostTags(List hostTags, List hostInClusterTags, String offeringTag) { + private boolean checkHostTags(List hostTags, List hostInClusterTags, String offeringTag) { if (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) { return true; } else if ((CollectionUtils.isNotEmpty(hostTags) && CollectionUtils.isEmpty(hostInClusterTags)) || (CollectionUtils.isEmpty(hostTags) && CollectionUtils.isNotEmpty(hostInClusterTags))) { return false; } else { - return hostInClusterTags.contains(offeringTag); + return hostInClusterTags.parallelStream().anyMatch(hostTagVO -> offeringTag.equals(hostTagVO.getTag())); } } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 494855b253d..66063bee0f7 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -854,6 +854,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C params.put("hypervisorType", hypervisorType); params.put("url", cmd.getUrl()); params.put("tags", cmd.getTags()); + params.put("isTagARule", cmd.isTagARule()); params.put("name", cmd.getStoragePoolName()); params.put("details", details); params.put("providerName", storeProvider.getName()); @@ -1017,10 +1018,10 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (pool.getPoolType() == StoragePoolType.DatastoreCluster) { List childStoragePools = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(pool.getId()); for (StoragePoolVO childPool : childStoragePools) { - _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags); + _storagePoolTagsDao.persist(childPool.getId(), storagePoolTags, cmd.isTagARule()); } } - _storagePoolTagsDao.persist(pool.getId(), storagePoolTags); + _storagePoolTagsDao.persist(pool.getId(), storagePoolTags, cmd.isTagARule()); } Long updatedCapacityBytes = null; @@ -1991,7 +1992,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List childDatastoreAnswerList, long hostId) { StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId); - List storageTags = _storagePoolTagsDao.getStoragePoolTags(datastoreClusterPoolId); + List storageTags = _storagePoolTagsDao.findStoragePoolTags(datastoreClusterPoolId); List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(datastoreClusterPoolId); Set childDatastoreUUIDs = new HashSet<>(); for (StoragePoolVO childDatastore : childDatastores) { @@ -2019,18 +2020,18 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C dataStoreVO.setParent(datastoreClusterPoolId); _storagePoolDao.update(dataStoreVO.getId(), dataStoreVO); if (CollectionUtils.isNotEmpty(storageTags)) { - storageTags.addAll(_storagePoolTagsDao.getStoragePoolTags(dataStoreVO.getId())); + storageTags.addAll(_storagePoolTagsDao.findStoragePoolTags(dataStoreVO.getId())); } else { - storageTags = _storagePoolTagsDao.getStoragePoolTags(dataStoreVO.getId()); + storageTags = _storagePoolTagsDao.findStoragePoolTags(dataStoreVO.getId()); } if (CollectionUtils.isNotEmpty(storageTags)) { - Set set = new LinkedHashSet<>(storageTags); + Set set = new LinkedHashSet<>(storageTags); storageTags.clear(); storageTags.addAll(set); if (s_logger.isDebugEnabled()) { s_logger.debug("Updating Storage Pool Tags to :" + storageTags); } - _storagePoolTagsDao.persist(dataStoreVO.getId(), storageTags); + _storagePoolTagsDao.persist(storageTags); } } else { // This is to find datastores which are removed from datastore cluster. @@ -2038,7 +2039,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C childDatastoreUUIDs.remove(dataStoreVO.getUuid()); } } else { - dataStoreVO = createChildDatastoreVO(datastoreClusterPool, childDataStoreAnswer); + dataStoreVO = createChildDatastoreVO(datastoreClusterPool, childDataStoreAnswer, storageTags); } updateStoragePoolHostVOAndBytes(dataStoreVO, hostId, childDataStoreAnswer); } @@ -2079,9 +2080,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C } } - private StoragePoolVO createChildDatastoreVO(StoragePoolVO datastoreClusterPool, ModifyStoragePoolAnswer childDataStoreAnswer) { + private StoragePoolVO createChildDatastoreVO(StoragePoolVO datastoreClusterPool, ModifyStoragePoolAnswer childDataStoreAnswer, List storagePoolTagVOList) { StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); - List storageTags = _storagePoolTagsDao.getStoragePoolTags(datastoreClusterPool.getId()); StoragePoolVO dataStoreVO = new StoragePoolVO(); dataStoreVO.setStorageProviderName(datastoreClusterPool.getStorageProviderName()); @@ -2108,7 +2108,15 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { details.put("pool_type", childDataStoreAnswer.getPoolType()); } - _storagePoolDao.persist(dataStoreVO, details, storageTags); + + List storagePoolTags = new ArrayList<>(); + boolean isTagARule = false; + if (CollectionUtils.isNotEmpty(storagePoolTagVOList)) { + storagePoolTags = storagePoolTagVOList.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()); + isTagARule = storagePoolTagVOList.get(0).isTagARule(); + } + + _storagePoolDao.persist(dataStoreVO, details, storagePoolTags, isTagARule); return dataStoreVO; } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 7bfe8a0818b..b4faf2ee479 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -106,6 +106,7 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; +import org.apache.cloudstack.utils.jsinterpreter.TagAsRuleHelper; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.cloudstack.utils.volume.VirtualMachineDiskInfo; import org.apache.commons.collections.CollectionUtils; @@ -346,6 +347,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Inject protected StoragePoolDetailsDao storagePoolDetailsDao; + protected Gson _gson; private static final List SupportedHypervisorsForVolResize = Arrays.asList(HypervisorType.KVM, HypervisorType.XenServer, @@ -375,6 +377,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic "Time (in milliseconds) to wait before assuming the VM was unable to detach a volume after the hypervisor sends the detach command.", true); + public static ConfigKey storageTagRuleExecutionTimeout = new ConfigKey<>("Advanced", Long.class, "storage.tag.rule.execution.timeout", "2000", "The maximum runtime," + + " in milliseconds, to execute a storage tag rule; if it is reached, a timeout will happen.", true); + private final StateMachine2 _volStateMachine; private static final Set STATES_VOLUME_CANNOT_BE_DESTROYED = new HashSet<>(Arrays.asList(Volume.State.Destroy, Volume.State.Expunging, Volume.State.Expunged, Volume.State.Allocated)); @@ -3266,7 +3271,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } if (!doesTargetStorageSupportDiskOffering(destPool, newDiskOffering)) { throw new InvalidParameterValueException(String.format("Migration failed: target pool [%s, tags:%s] has no matching tags for volume [%s, uuid:%s, tags:%s]", destPool.getName(), - getStoragePoolTags(destPool), volume.getName(), volume.getUuid(), newDiskOffering.getTags())); + storagePoolTagsDao.getStoragePoolTags(destPool.getId()), volume.getName(), volume.getUuid(), newDiskOffering.getTags())); } if (volume.getVolumeType().equals(Volume.Type.ROOT)) { VMInstanceVO vm = null; @@ -3329,17 +3334,31 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic @Override public boolean doesTargetStorageSupportDiskOffering(StoragePool destPool, String diskOfferingTags) { - if (StringUtils.isBlank(diskOfferingTags)) { + Pair, Boolean> storagePoolTags = getStoragePoolTags(destPool); + if ((storagePoolTags == null || !storagePoolTags.second()) && org.apache.commons.lang.StringUtils.isBlank(diskOfferingTags)) { + if (storagePoolTags == null) { + s_logger.debug(String.format("Destination storage pool [%s] does not have any tags, and so does the disk offering. Therefore, they are compatible", destPool.getUuid())); + } else { + s_logger.debug("Destination storage pool has tags [%s], and the disk offering has no tags. Therefore, they are compatible."); + } return true; } - String storagePoolTags = getStoragePoolTags(destPool); - if (StringUtils.isBlank(storagePoolTags)) { + if (storagePoolTags == null || CollectionUtils.isEmpty(storagePoolTags.first())) { + s_logger.debug(String.format("Destination storage pool [%s] has no tags, while disk offering has tags [%s]. Therefore, they are not compatible", destPool.getUuid(), + diskOfferingTags)); return false; } - String[] storageTagsAsStringArray = StringUtils.split(storagePoolTags, ","); - String[] newDiskOfferingTagsAsStringArray = StringUtils.split(diskOfferingTags, ","); + List storageTagsList = storagePoolTags.first(); + String[] newDiskOfferingTagsAsStringArray = org.apache.commons.lang.StringUtils.split(diskOfferingTags, ","); - return CollectionUtils.isSubCollection(Arrays.asList(newDiskOfferingTagsAsStringArray), Arrays.asList(storageTagsAsStringArray)); + boolean result; + if (storagePoolTags.second()) { + result = TagAsRuleHelper.interpretTagAsRule(storageTagsList.get(0), diskOfferingTags, storageTagRuleExecutionTimeout.value()); + } else { + result = CollectionUtils.isSubCollection(Arrays.asList(newDiskOfferingTagsAsStringArray), storageTagsList); + } + s_logger.debug(String.format("Destination storage pool [%s] accepts tags [%s]? %s", destPool.getUuid(), diskOfferingTags, result)); + return result; } public static boolean doesNewDiskOfferingHasTagsAsOldDiskOffering(DiskOfferingVO oldDO, DiskOfferingVO newDO) { @@ -3355,14 +3374,17 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic } /** - * Retrieves the storage pool tags as a {@link String}. If the storage pool does not have tags we return a null value. + * Returns a {@link Pair}, where the first value is the list of the StoragePool tags, and the second value is whether the returned tags are to be interpreted as a rule, + * or a normal list of tags. + *

+ * If the storage pool does not have tags we return a null value. */ - protected String getStoragePoolTags(StoragePool destPool) { - List destPoolTags = storagePoolTagsDao.getStoragePoolTags(destPool.getId()); + protected Pair, Boolean> getStoragePoolTags(StoragePool destPool) { + List destPoolTags = storagePoolTagsDao.findStoragePoolTags(destPool.getId()); if (CollectionUtils.isEmpty(destPoolTags)) { return null; } - return StringUtils.join(destPoolTags, ","); + return new Pair<>(destPoolTags.parallelStream().map(StoragePoolTagVO::getTag).collect(Collectors.toList()), destPoolTags.get(0).isTagARule()); } private Volume orchestrateMigrateVolume(VolumeVO volume, StoragePool destPool, boolean liveMigrateVolume, DiskOfferingVO newDiskOffering) { diff --git a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java index 63ae60411ab..d6101046383 100644 --- a/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java +++ b/server/src/main/java/com/cloud/storage/listener/StoragePoolMonitor.java @@ -101,7 +101,7 @@ public class StoragePoolMonitor implements Listener { scCmd.getHypervisorType() == HypervisorType.Ovm || scCmd.getHypervisorType() == HypervisorType.Hyperv || scCmd.getHypervisorType() == HypervisorType.LXC || scCmd.getHypervisorType() == HypervisorType.Ovm3) { List pools = _poolDao.listBy(host.getDataCenterId(), host.getPodId(), host.getClusterId(), ScopeType.CLUSTER); - List zoneStoragePoolsByTags = _poolDao.findZoneWideStoragePoolsByTags(host.getDataCenterId(), null); + List zoneStoragePoolsByTags = _poolDao.findZoneWideStoragePoolsByTags(host.getDataCenterId(), null, false); List zoneStoragePoolsByHypervisor = _poolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), scCmd.getHypervisorType()); zoneStoragePoolsByTags.retainAll(zoneStoragePoolsByHypervisor); pools.addAll(zoneStoragePoolsByTags); diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java index 8493535cf85..c2d748ee587 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerTest.java @@ -58,6 +58,7 @@ import com.cloud.network.dao.PhysicalNetworkDao; import com.cloud.network.dao.PhysicalNetworkVO; import com.cloud.projects.ProjectManager; import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.StoragePoolTagVO; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; @@ -1163,12 +1164,17 @@ public class ConfigurationManagerTest { @Test public void updateDiskOfferingTagsWithPrimaryStorageWithCorrectTagsTestSuccess(){ String tags = "tag1,tag2"; - List storageTagsWithCorrectTags = new ArrayList<>(Arrays.asList("tag1","tag2")); List pools = new ArrayList<>(Arrays.asList(storagePoolVO)); List volumes = new ArrayList<>(Arrays.asList(volumeVO)); + StoragePoolTagVO poolTagMock1 = Mockito.mock(StoragePoolTagVO.class); + StoragePoolTagVO poolTagMock2 = Mockito.mock(StoragePoolTagVO.class); + List poolTags = List.of(poolTagMock1, poolTagMock2); + Mockito.doReturn("tag1").when(poolTagMock1).getTag(); + Mockito.doReturn("tag2").when(poolTagMock2).getTag(); + Mockito.when(primaryDataStoreDao.listStoragePoolsWithActiveVolumesByOfferingId(anyLong())).thenReturn(pools); - Mockito.when(storagePoolTagsDao.getStoragePoolTags(anyLong())).thenReturn(storageTagsWithCorrectTags); + Mockito.when(storagePoolTagsDao.findStoragePoolTags(anyLong())).thenReturn(poolTags); Mockito.when(diskOfferingDao.findById(anyLong())).thenReturn(diskOfferingVOMock); Mockito.when(_volumeDao.findByDiskOfferingId(anyLong())).thenReturn(volumes); diff --git a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java index 8ea14846bef..d56a223082a 100644 --- a/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java +++ b/server/src/test/java/com/cloud/storage/VolumeApiServiceImplTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.when; import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.UUID; @@ -117,6 +118,7 @@ import com.cloud.user.ResourceLimitService; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.user.dao.AccountDao; +import com.cloud.utils.Pair; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.fsm.NoTransitionException; @@ -650,29 +652,35 @@ public class VolumeApiServiceImplTest { @Test public void getStoragePoolTagsTestStorageWithoutTags() { - Mockito.when(storagePoolTagsDao.getStoragePoolTags(storagePoolMockId)).thenReturn(new ArrayList<>()); - - String returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); + Pair, Boolean> returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); Assert.assertNull(returnedStoragePoolTags); } @Test public void getStoragePoolTagsTestStorageWithTags() { - ArrayList tags = new ArrayList<>(); - String tag1 = "tag1"; - String tag2 = "tag2"; - String tag3 = "tag3"; + StoragePoolTagVO tag1 = new StoragePoolTagVO(1,"tag1", false); + StoragePoolTagVO tag2 = new StoragePoolTagVO(1,"tag2", false); + StoragePoolTagVO tag3 = new StoragePoolTagVO(1,"tag3", false); + List tags = Arrays.asList(tag1, tag2, tag3); - tags.add(tag1); - tags.add(tag2); - tags.add(tag3); + Mockito.when(storagePoolTagsDao.findStoragePoolTags(storagePoolMockId)).thenReturn(tags); - Mockito.when(storagePoolTagsDao.getStoragePoolTags(storagePoolMockId)).thenReturn(tags); + Pair, Boolean> returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); - String returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); + Assert.assertEquals(new Pair<>(Arrays.asList("tag1","tag2","tag3"), false), returnedStoragePoolTags); + } - Assert.assertEquals("tag1,tag2,tag3", returnedStoragePoolTags); + @Test + public void getStoragePoolTagsTestStorageWithRuleTag() { + StoragePoolTagVO tag1 = new StoragePoolTagVO(1,"tag1", true); + List tags = List.of(tag1); + + Mockito.when(storagePoolTagsDao.findStoragePoolTags(storagePoolMockId)).thenReturn(tags); + + Pair, Boolean> returnedStoragePoolTags = volumeApiServiceImpl.getStoragePoolTags(storagePoolMock); + + Assert.assertEquals(new Pair<>(List.of("tag1"), true), returnedStoragePoolTags); } @Test @@ -757,7 +765,7 @@ public class VolumeApiServiceImplTest { Mockito.when(newDiskOfferingMock.getTags()).thenReturn("tag1"); - Mockito.doReturn("tag1").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("tag1"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); volumeApiServiceImpl.validateConditionsToReplaceDiskOfferingOfVolume(volumeVoMock, newDiskOfferingMock, storagePoolMock); @@ -1138,7 +1146,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A,B,C").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("A").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("A"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1151,7 +1159,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A,B,C").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("A,B,C,D,X,Y").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("A","B","C","D","X","Y"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1164,7 +1172,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.lenient().doReturn("A,B,C,D,X,Y").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.lenient().doReturn(new Pair<>(List.of("A,B,C,D,X,Y"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1177,7 +1185,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of(""), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1190,7 +1198,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.lenient().doReturn("").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.lenient().doReturn(new Pair<>(List.of(""), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1203,7 +1211,7 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A,B").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("C,D").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("C,D"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); @@ -1216,13 +1224,52 @@ public class VolumeApiServiceImplTest { Mockito.doReturn("A").when(diskOfferingVoMock).getTags(); StoragePool storagePoolMock = Mockito.mock(StoragePool.class); - Mockito.doReturn("A").when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + Mockito.doReturn(new Pair<>(List.of("A"), false)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); Assert.assertTrue(result); } + @Test + public void doesTargetStorageSupportDiskOfferingTestStorageRuleTagWithDiskOfferingTagThatMatches() { + DiskOfferingVO diskOfferingVoMock = Mockito.mock(DiskOfferingVO.class); + Mockito.doReturn("A").when(diskOfferingVoMock).getTags(); + + StoragePool storagePoolMock = Mockito.mock(StoragePool.class); + Mockito.doReturn(new Pair<>(List.of("tags[0] == 'A'"), true)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + + boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); + + Assert.assertTrue(result); + } + + @Test + public void doesTargetStorageSupportDiskOfferingTestStorageRuleTagWithDiskOfferingTagThatDoesNotMatch() { + DiskOfferingVO diskOfferingVoMock = Mockito.mock(DiskOfferingVO.class); + Mockito.doReturn("'").when(diskOfferingVoMock).getTags(); + + StoragePool storagePoolMock = Mockito.mock(StoragePool.class); + Mockito.doReturn(new Pair<>(List.of("tags[0] == 'A'"), true)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + + boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); + + Assert.assertFalse(result); + } + + @Test + public void doesTargetStorageSupportDiskOfferingTestStorageRuleTagWithNullDiskOfferingTag() { + DiskOfferingVO diskOfferingVoMock = Mockito.mock(DiskOfferingVO.class); + Mockito.doReturn(null).when(diskOfferingVoMock).getTags(); + + StoragePool storagePoolMock = Mockito.mock(StoragePool.class); + Mockito.doReturn(new Pair<>(List.of("tags[0] == 'A'"), true)).when(volumeApiServiceImpl).getStoragePoolTags(storagePoolMock); + + boolean result = volumeApiServiceImpl.doesTargetStorageSupportDiskOffering(storagePoolMock, diskOfferingVoMock); + + Assert.assertFalse(result); + } + @Test public void validateIfVmHaveBackupsTestExceptionWhenTryToDetachVolumeFromVMWhichBackupOffering() { try { diff --git a/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java b/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java index b892b80c9d4..fa6b71d0cb2 100644 --- a/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java +++ b/server/src/test/java/com/cloud/storage/listener/StoragePoolMonitorTest.java @@ -60,7 +60,7 @@ public class StoragePoolMonitorTest { @Test public void testProcessConnectStoragePoolNormal() throws Exception { Mockito.when(poolDao.listBy(nullable(Long.class), nullable(Long.class), nullable(Long.class), Mockito.any(ScopeType.class))).thenReturn(Collections.singletonList(pool)); - Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class))).thenReturn(Collections.emptyList()); + Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class), Mockito.anyBoolean())).thenReturn(Collections.emptyList()); Mockito.when(poolDao.findZoneWideStoragePoolsByHypervisor(Mockito.anyLong(), Mockito.any(Hypervisor.HypervisorType.class))).thenReturn(Collections.emptyList()); Mockito.doReturn(true).when(storageManager).connectHostToSharedPool(host.getId(), pool.getId()); @@ -73,7 +73,7 @@ public class StoragePoolMonitorTest { @Test public void testProcessConnectStoragePoolFailureOnHost() throws Exception { Mockito.when(poolDao.listBy(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(ScopeType.class))).thenReturn(Collections.singletonList(pool)); - Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class))).thenReturn(Collections.emptyList()); + Mockito.when(poolDao.findZoneWideStoragePoolsByTags(Mockito.anyLong(), Mockito.any(String[].class), Mockito.anyBoolean())).thenReturn(Collections.emptyList()); Mockito.when(poolDao.findZoneWideStoragePoolsByHypervisor(Mockito.anyLong(), Mockito.any(Hypervisor.HypervisorType.class))).thenReturn(Collections.emptyList()); Mockito.doThrow(new StorageUnavailableException("unable to mount storage", 123L)).when(storageManager).connectHostToSharedPool(Mockito.anyLong(), Mockito.anyLong()); diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 199437c8033..17da726ae45 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1081,6 +1081,7 @@ "label.isdedicated": "Dedicated", "label.isdefault": "Is default", "label.isdynamicallyscalable": "Dynamically scalable", +"label.istagarule": "Tag as JS rule", "label.isextractable": "Extractable", "label.isfeatured": "Featured", "label.isforced": "Force delete", diff --git a/ui/public/locales/pt_BR.json b/ui/public/locales/pt_BR.json index 5b275a12602..c36a5c762a7 100644 --- a/ui/public/locales/pt_BR.json +++ b/ui/public/locales/pt_BR.json @@ -844,6 +844,7 @@ "label.isdedicated": "Dedicado", "label.isdefault": "\u00c9\u0089 padr\u00e3o", "label.isdynamicallyscalable": "Dinamicamente escal\u00e1vel", +"label.istagarule": "Tag como regra JS", "label.isextractable": "Extra\u00edvel", "label.isfeatured": "Em destaque", "label.isforced": "For\u00e7ar exclus\u00e3o", diff --git a/ui/src/config/section/infra/hosts.js b/ui/src/config/section/infra/hosts.js index af3ad1dbcb5..9a6fc021152 100644 --- a/ui/src/config/section/infra/hosts.js +++ b/ui/src/config/section/infra/hosts.js @@ -68,7 +68,7 @@ export default { icon: 'edit-outlined', label: 'label.edit', dataView: true, - args: ['name', 'hosttags', 'oscategoryid'], + args: ['name', 'hosttags', 'istagarule', 'oscategoryid'], mapping: { oscategoryid: { api: 'listOsCategories' diff --git a/ui/src/config/section/infra/primaryStorages.js b/ui/src/config/section/infra/primaryStorages.js index 6f51c7bd0a3..f222edeaf70 100644 --- a/ui/src/config/section/infra/primaryStorages.js +++ b/ui/src/config/section/infra/primaryStorages.js @@ -89,7 +89,7 @@ export default { icon: 'edit-outlined', label: 'label.edit', dataView: true, - args: ['name', 'tags', 'capacitybytes', 'capacityiops'] + args: ['name', 'tags', 'istagarule', 'capacitybytes', 'capacityiops'] }, { api: 'updateStoragePool', diff --git a/utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/TagAsRuleHelper.java b/utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/TagAsRuleHelper.java new file mode 100644 index 00000000000..114818afc93 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/jsinterpreter/TagAsRuleHelper.java @@ -0,0 +1,51 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.utils.jsinterpreter; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.log4j.Logger; + +import java.io.IOException; + +public class TagAsRuleHelper { + + private static final Logger LOGGER = Logger.getLogger(TagAsRuleHelper.class); + + private static final String PARSE_TAGS = "tags = tags ? tags.split(',') : [];"; + + + public static boolean interpretTagAsRule(String rule, String tags, long timeout) { + String script = PARSE_TAGS + rule; + tags = String.format("'%s'", StringEscapeUtils.escapeEcmaScript(tags)); + try (JsInterpreter jsInterpreter = new JsInterpreter(timeout)) { + jsInterpreter.injectVariable("tags", tags); + Object scriptReturn = jsInterpreter.executeScript(script); + if (scriptReturn instanceof Boolean) { + return (Boolean)scriptReturn; + } + } catch (IOException ex) { + String message = String.format("Error while executing script [%s].", script); + LOGGER.error(message, ex); + throw new CloudRuntimeException(message, ex); + } + + LOGGER.debug(String.format("Result of tag rule [%s] was not a boolean, returning false.", script)); + return false; + } + +} From 98cd3b9a24332c4de147f39b8da346de45cb4952 Mon Sep 17 00:00:00 2001 From: Oleg Chuev <91337563+OlegChuev@users.noreply.github.com> Date: Thu, 30 Nov 2023 14:46:07 +0200 Subject: [PATCH 12/50] UI: Removed ICMP input fields for protocol number from ACL List rules modal (#8253) --- ui/src/views/network/AclListRulesTab.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/views/network/AclListRulesTab.vue b/ui/src/views/network/AclListRulesTab.vue index 36f10e98b68..c8b6689fb59 100644 --- a/ui/src/views/network/AclListRulesTab.vue +++ b/ui/src/views/network/AclListRulesTab.vue @@ -212,7 +212,7 @@ -
+
From cb62ce67671699fa01564b3b4b0d3d83eb3d5acb Mon Sep 17 00:00:00 2001 From: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> Date: Thu, 30 Nov 2023 10:51:43 -0300 Subject: [PATCH 13/50] Global ACL for VPCs (#7150) --- .github/workflows/ci.yml | 3 +- .../user/network/CreateNetworkACLListCmd.java | 36 ++- .../com/cloud/network/vpc/NetworkACLVO.java | 7 + .../com/cloud/network/NetworkServiceImpl.java | 17 +- .../network/vpc/NetworkACLServiceImpl.java | 133 ++++++---- .../com/cloud/network/vpc/VpcManagerImpl.java | 14 +- .../vpc/NetworkACLServiceImplTest.java | 153 +++++++---- test/integration/smoke/test_global_acls.py | 245 ++++++++++++++++++ 8 files changed, 494 insertions(+), 114 deletions(-) create mode 100644 test/integration/smoke/test_global_acls.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6b5f5a1c4f..77c9d59505c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,8 @@ jobs: smoke/test_reset_configuration_settings smoke/test_reset_vm_on_reboot smoke/test_resource_accounting - smoke/test_resource_detail", + smoke/test_resource_detail + smoke/test_global_acls", "smoke/test_router_dhcphosts smoke/test_router_dns smoke/test_router_dnsservice diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java index 14dbfcafd7b..e5dbcc7b6d1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkACLListCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.network; +import com.cloud.exception.PermissionDeniedException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -26,6 +27,7 @@ import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.NetworkACLResponse; import org.apache.cloudstack.api.response.VpcResponse; +import org.apache.cloudstack.context.CallContext; import org.apache.log4j.Logger; import com.cloud.event.EventTypes; @@ -35,7 +37,8 @@ import com.cloud.network.vpc.NetworkACL; import com.cloud.network.vpc.Vpc; import com.cloud.user.Account; -@APICommand(name = "createNetworkACLList", description = "Creates a network ACL for the given VPC", responseObject = NetworkACLResponse.class, +@APICommand(name = "createNetworkACLList", description = "Creates a network ACL. If no VPC is given, then it creates a global ACL that can be used by everyone.", + responseObject = NetworkACLResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { public static final Logger s_logger = Logger.getLogger(CreateNetworkACLListCmd.class.getName()); @@ -53,7 +56,6 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.VPC_ID, type = CommandType.UUID, - required = true, entityType = VpcResponse.class, description = "ID of the VPC associated with this network ACL list") private Long vpcId; @@ -77,6 +79,10 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { return vpcId; } + public void setVpcId(Long vpcId) { + this.vpcId = vpcId; + } + @Override public boolean isDisplay() { if (display != null) { @@ -92,6 +98,9 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { @Override public void create() { + if (getVpcId() == null) { + setVpcId(0L); + } NetworkACL result = _networkACLService.createNetworkACL(getName(), getDescription(), getVpcId(), isDisplay()); setEntityId(result.getId()); setEntityUuid(result.getUuid()); @@ -111,12 +120,21 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { @Override public long getEntityOwnerId() { - Vpc vpc = _entityMgr.findById(Vpc.class, getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Invalid vpcId is given"); - } + Account account; + if (isAclAttachedToVpc(this.vpcId)) { + Vpc vpc = _entityMgr.findById(Vpc.class, this.vpcId); + if (vpc == null) { + throw new InvalidParameterValueException(String.format("Invalid VPC ID [%s] provided.", this.vpcId)); + } + account = _accountService.getAccount(vpc.getAccountId()); + } else { + account = CallContext.current().getCallingAccount(); + if (!Account.Type.ADMIN.equals(account.getType())) { + s_logger.warn(String.format("Only Root Admin can create global ACLs. Account [%s] cannot create any global ACL.", account)); + throw new PermissionDeniedException("Only Root Admin can create global ACLs."); + } - Account account = _accountService.getAccount(vpc.getAccountId()); + } return account.getId(); } @@ -139,4 +157,8 @@ public class CreateNetworkACLListCmd extends BaseAsyncCreateCmd { public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.NetworkAcl; } + + public boolean isAclAttachedToVpc(Long aclVpcId) { + return aclVpcId != null && aclVpcId != 0; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java index 4eaa2b575e0..280d5dfaf4b 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/NetworkACLVO.java @@ -17,6 +17,8 @@ package com.cloud.network.vpc; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + import java.util.UUID; import javax.persistence.Column; @@ -85,6 +87,11 @@ public class NetworkACLVO implements NetworkACL { return name; } + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "uuid", "name", "vpcId"); + } + public void setUuid(String uuid) { this.uuid = uuid; } diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index f9f80cb5b76..18f98e6f99f 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -2100,12 +2100,9 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C throw new InvalidParameterValueException("Unable to find specified NetworkACL"); } - if (aclId != NetworkACL.DEFAULT_DENY && aclId != NetworkACL.DEFAULT_ALLOW) { - // ACL is not default DENY/ALLOW - // ACL should be associated with a VPC - if (!vpcId.equals(acl.getVpcId())) { - throw new InvalidParameterValueException("ACL: " + aclId + " do not belong to the VPC"); - } + Long aclVpcId = acl.getVpcId(); + if (!isDefaultAcl(aclId) && isAclAttachedToVpc(aclVpcId, vpcId)) { + throw new InvalidParameterValueException(String.format("ACL [%s] does not belong to the VPC [%s].", aclId, aclVpcId)); } } network = _vpcMgr.createVpcGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, networkDomain, owner, sharedDomainId, pNtwk, zoneId, aclType, @@ -5950,6 +5947,14 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C return new ConfigKey[] {AllowDuplicateNetworkName, AllowEmptyStartEndIpAddress, VRPrivateInterfaceMtu, VRPublicInterfaceMtu, AllowUsersToSpecifyVRMtu}; } + public boolean isDefaultAcl(Long aclId) { + return aclId == NetworkACL.DEFAULT_DENY || aclId == NetworkACL.DEFAULT_ALLOW; + } + + public boolean isAclAttachedToVpc(Long aclVpcId, Long vpcId) { + return aclVpcId != 0 && !vpcId.equals(aclVpcId); + } + @Override public PublicIpQuarantine updatePublicIpAddressInQuarantine(UpdateQuarantinedIpCmd cmd) throws CloudRuntimeException { Long ipId = cmd.getId(); diff --git a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java index ed37eb5b375..8139ac1c49e 100644 --- a/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/NetworkACLServiceImpl.java @@ -24,6 +24,7 @@ import java.util.Map; import javax.inject.Inject; +import com.cloud.exception.PermissionDeniedException; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; @@ -103,12 +104,15 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ @Override public NetworkACL createNetworkACL(final String name, final String description, final long vpcId, final Boolean forDisplay) { - final Account caller = CallContext.current().getCallingAccount(); - final Vpc vpc = _entityMgr.findById(Vpc.class, vpcId); - if (vpc == null) { - throw new InvalidParameterValueException("Unable to find VPC"); + if (vpcId != 0) { + final Account caller = CallContext.current().getCallingAccount(); + final Vpc vpc = _vpcDao.findById(vpcId); + if (vpc == null) { + throw new InvalidParameterValueException(String.format("Unable to find VPC with ID [%s].", vpcId)); + } + _accountMgr.checkAccess(caller, null, true, vpc); + } - _accountMgr.checkAccess(caller, null, true, vpc); return _networkAclMgr.createNetworkACL(name, description, vpcId, forDisplay); } @@ -212,22 +216,17 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ @Override @ActionEvent(eventType = EventTypes.EVENT_NETWORK_ACL_DELETE, eventDescription = "Deleting Network ACL List", async = true) public boolean deleteNetworkACL(final long id) { - final Account caller = CallContext.current().getCallingAccount(); final NetworkACL acl = _networkACLDao.findById(id); + Account account = CallContext.current().getCallingAccount(); if (acl == null) { throw new InvalidParameterValueException("Unable to find specified ACL"); } - //Do not allow deletion of default ACLs - if (acl.getId() == NetworkACL.DEFAULT_ALLOW || acl.getId() == NetworkACL.DEFAULT_DENY) { + if (isDefaultAcl(acl.getId())) { throw new InvalidParameterValueException("Default ACL cannot be removed"); } - final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Unable to find specified VPC associated with the ACL"); - } - _accountMgr.checkAccess(caller, null, true, vpc); + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admin can delete global ACLs."); return _networkAclMgr.deleteNetworkACL(acl); } @@ -253,7 +252,7 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ throw new InvalidParameterValueException("Unable to find specified vpc id"); } - if (aclId != NetworkACL.DEFAULT_DENY && aclId != NetworkACL.DEFAULT_ALLOW) { + if (!isDefaultAcl(aclId)) { final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); if (vpc == null) { throw new InvalidParameterValueException("Unable to find Vpc associated with the NetworkACL"); @@ -293,15 +292,9 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ throw new InvalidParameterValueException("Network ACL can be created just for networks of type " + Networks.TrafficType.Guest); } - if (aclId != NetworkACL.DEFAULT_DENY && aclId != NetworkACL.DEFAULT_ALLOW) { - //ACL is not default DENY/ALLOW - // ACL should be associated with a VPC - final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Unable to find Vpc associated with the NetworkACL"); - } + if (!isDefaultAcl(aclId) && !isGlobalAcl(acl.getVpcId())) { + validateAclAssociatedToVpc(acl.getVpcId(), caller, acl.getUuid()); - _accountMgr.checkAccess(caller, null, true, vpc); if (!network.getVpcId().equals(acl.getVpcId())) { throw new InvalidParameterValueException("Network: " + networkId + " and ACL: " + aclId + " do not belong to the same VPC"); } @@ -340,6 +333,11 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ NetworkACL acl = _networkAclMgr.getNetworkACL(aclId); validateNetworkAcl(acl); + Account caller = CallContext.current().getCallingAccount(); + + if (isGlobalAcl(acl.getVpcId()) && !Account.Type.ADMIN.equals(caller.getType())) { + throw new PermissionDeniedException("Only Root Admins can create rules for a global ACL."); + } validateAclRuleNumber(createNetworkACLCmd, acl); NetworkACLItem.Action ruleAction = validateAndCreateNetworkAclRuleAction(action); @@ -409,7 +407,8 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ *
    *
  • If the parameter is null, we return an {@link InvalidParameterValueException}; *
  • Default ACLs {@link NetworkACL#DEFAULT_ALLOW} and {@link NetworkACL#DEFAULT_DENY} cannot be modified. Therefore, if any of them is provided we throw a {@link InvalidParameterValueException}; - *
  • If the network does not have a VPC, we will throw an {@link InvalidParameterValueException}. + *
  • If no VPC is given, then it is a global ACL and there is no need to check any VPC ID. However, if a VPC is given and it does not exist, throws an + * {@link InvalidParameterValueException}. *
* * After all validations, we check if the user has access to the given network ACL using {@link AccountManager#checkAccess(Account, org.apache.cloudstack.acl.SecurityChecker.AccessType, boolean, org.apache.cloudstack.acl.ControlledEntity...)}. @@ -419,16 +418,14 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ throw new InvalidParameterValueException("Unable to find specified ACL."); } - if (acl.getId() == NetworkACL.DEFAULT_DENY || acl.getId() == NetworkACL.DEFAULT_ALLOW) { + if (isDefaultAcl(acl.getId())) { throw new InvalidParameterValueException("Default ACL cannot be modified"); } - Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException(String.format("Unable to find Vpc associated with the NetworkACL [%s]", acl.getUuid())); + Long aclVpcId = acl.getVpcId(); + if (!isGlobalAcl(aclVpcId)) { + validateAclAssociatedToVpc(aclVpcId, CallContext.current().getCallingAccount(), acl.getUuid()); } - Account caller = CallContext.current().getCallingAccount(); - _accountMgr.checkAccess(caller, null, true, vpc); } /** @@ -792,17 +789,12 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ final NetworkACLItemVO aclItem = _networkACLItemDao.findById(ruleId); if (aclItem != null) { final NetworkACL acl = _networkAclMgr.getNetworkACL(aclItem.getAclId()); + final Account account = CallContext.current().getCallingAccount(); - final Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - - if (aclItem.getAclId() == NetworkACL.DEFAULT_ALLOW || aclItem.getAclId() == NetworkACL.DEFAULT_DENY) { + if (isDefaultAcl(aclItem.getAclId())) { throw new InvalidParameterValueException("ACL Items in default ACL cannot be deleted"); } - - final Account caller = CallContext.current().getCallingAccount(); - - _accountMgr.checkAccess(caller, null, true, vpc); - + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admin can delete global ACL rules."); } return _networkAclMgr.revokeNetworkACLItem(ruleId); } @@ -826,6 +818,9 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ NetworkACL acl = _networkAclMgr.getNetworkACL(networkACLItemVo.getAclId()); validateNetworkAcl(acl); + Account account = CallContext.current().getCallingAccount(); + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Only Root Admins can update global ACLs."); + transferDataToNetworkAclRulePojo(updateNetworkACLItemCmd, networkACLItemVo, acl); validateNetworkACLItem(networkACLItemVo); return _networkAclMgr.updateNetworkACLItem(networkACLItemVo); @@ -912,14 +907,13 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ } @Override - @ActionEvent(eventType = EventTypes.EVENT_NETWORK_ACL_UPDATE, eventDescription = "updating network acl", async = true) + @ActionEvent(eventType = EventTypes.EVENT_NETWORK_ACL_UPDATE, eventDescription = "Updating Network ACL", async = true) public NetworkACL updateNetworkACL(UpdateNetworkACLListCmd updateNetworkACLListCmd) { Long id = updateNetworkACLListCmd.getId(); NetworkACLVO acl = _networkACLDao.findById(id); - Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); + Account account = CallContext.current().getCallingAccount(); - Account caller = CallContext.current().getCallingAccount(); - _accountMgr.checkAccess(caller, null, true, vpc); + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account, "Must be Root Admin to update a global ACL."); String name = updateNetworkACLListCmd.getName(); if (StringUtils.isNotBlank(name)) { @@ -1149,14 +1143,59 @@ public class NetworkACLServiceImpl extends ManagerBase implements NetworkACLServ long aclId = ruleBeingMoved.getAclId(); if ((nextRule != null && nextRule.getAclId() != aclId) || (previousRule != null && previousRule.getAclId() != aclId)) { - throw new InvalidParameterValueException("Cannot use ACL rules from differenting ACLs. Rule being moved."); + throw new InvalidParameterValueException("Cannot use ACL rules from differentiating ACLs. Rule being moved."); } NetworkACLVO acl = _networkACLDao.findById(aclId); - Vpc vpc = _entityMgr.findById(Vpc.class, acl.getVpcId()); - if (vpc == null) { - throw new InvalidParameterValueException("Re-ordering rules for a default ACL is prohibited"); + Account account = CallContext.current().getCallingAccount(); + + if (isDefaultAcl(aclId)) { + throw new InvalidParameterValueException("Default ACL rules cannot be moved."); } - Account caller = CallContext.current().getCallingAccount(); - _accountMgr.checkAccess(caller, null, true, vpc); + + validateGlobalAclPermissionAndAclAssociatedToVpc(acl, account,"Must be Root Admin to move global ACL rules."); + } + + /** + * Checks if the given ACL is a global ACL. If it is, then checks if the account is Root Admin, and throws an exception according to {@code exceptionMessage} param if it + * does not have permission. + */ + protected void checkGlobalAclPermission(Long aclVpcId, Account account, String exceptionMessage) { + if (isGlobalAcl(aclVpcId) && !Account.Type.ADMIN.equals(account.getType())) { + throw new PermissionDeniedException(exceptionMessage); + } + } + + protected void validateAclAssociatedToVpc(Long aclVpcId, Account account, String aclUuid) { + final Vpc vpc = _vpcDao.findById(aclVpcId); + if (vpc == null) { + throw new InvalidParameterValueException(String.format("Unable to find specified VPC [%s] associated with the ACL [%s].", aclVpcId, aclUuid)); + } + _accountMgr.checkAccess(account, null, true, vpc); + } + + /** + * It performs two different verifications depending on if the ACL is global or not. + *
    + *
  • If the given ACL is a Global ACL, i.e. has VPC ID = 0, then checks if the account is Root Admin, and throws an exception if it isn't. + *
  • If the given ACL is associated to a VPC, i.e. VPC ID != 0, then calls {@link #validateAclAssociatedToVpc(Long, Account, String)} and checks if the given {@code + * aclVpcId} is from a valid VPC. + *
+ */ + protected void validateGlobalAclPermissionAndAclAssociatedToVpc(NetworkACL acl, Account account, String exception){ + if (isGlobalAcl(acl.getVpcId())) { + s_logger.info(String.format("Checking if account [%s] has permission to manipulate global ACL [%s].", account, acl)); + checkGlobalAclPermission(acl.getVpcId(), account, exception); + } else { + s_logger.info(String.format("Validating ACL [%s] associated to VPC [%s] with account [%s].", acl, acl.getVpcId(), account)); + validateAclAssociatedToVpc(acl.getVpcId(), account, acl.getUuid()); + } + } + + protected boolean isGlobalAcl(Long aclVpcId) { + return aclVpcId != null && aclVpcId == 0; + } + + protected boolean isDefaultAcl(long aclId) { + return aclId == NetworkACL.DEFAULT_ALLOW || aclId == NetworkACL.DEFAULT_DENY; } } diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java index 6bc70a9cef4..1f99d164625 100644 --- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java @@ -1977,16 +1977,14 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis searchBuilder.and("vpcId", searchBuilder.entity().getVpcId(), Op.IN); final SearchCriteria searchCriteria = searchBuilder.create(); - searchCriteria.setParameters("vpcId", vpcId, 0); + searchCriteria.setParameters("vpcId", vpcId); final Filter filter = new Filter(NetworkACLVO.class, "id", false, null, null); final Pair, Integer> aclsCountPair = _networkAclDao.searchAndCount(searchCriteria, filter); final List acls = aclsCountPair.first(); for (final NetworkACLVO networkAcl : acls) { - if (networkAcl.getId() != NetworkACL.DEFAULT_ALLOW && networkAcl.getId() != NetworkACL.DEFAULT_DENY) { - _networkAclMgr.deleteNetworkACL(networkAcl); - } + _networkAclMgr.deleteNetworkACL(networkAcl); } VpcVO vpc = vpcDao.findById(vpcId); @@ -3175,4 +3173,12 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis } return filteredDomainIds; } + + protected boolean isGlobalAcl(Long aclVpcId) { + return aclVpcId != null && aclVpcId == 0; + } + + protected boolean isDefaultAcl(long aclId) { + return aclId == NetworkACL.DEFAULT_ALLOW || aclId == NetworkACL.DEFAULT_DENY; + } } diff --git a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java index 8dd3f32d6c4..18a072172ad 100644 --- a/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/NetworkACLServiceImplTest.java @@ -17,6 +17,40 @@ package com.cloud.network.vpc; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.times; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.cloud.exception.PermissionDeniedException; +import com.cloud.network.vpc.dao.VpcDao; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; +import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd; +import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLItemCmd; +import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLListCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceUnavailableException; import com.cloud.network.Network; @@ -30,40 +64,9 @@ import com.cloud.user.AccountManager; import com.cloud.user.User; import com.cloud.utils.db.EntityManager; import com.cloud.utils.exception.CloudRuntimeException; -import org.apache.cloudstack.acl.SecurityChecker.AccessType; -import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; -import org.apache.cloudstack.api.command.user.network.MoveNetworkAclItemCmd; -import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLItemCmd; -import org.apache.cloudstack.api.command.user.network.UpdateNetworkACLListCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.commons.lang3.StringUtils; import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InOrder; -import org.mockito.InjectMocks; -import org.mockito.Mock; import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.Spy; -import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; -import org.mockito.stubbing.Answer; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.times; @RunWith(MockitoJUnitRunner.class) public class NetworkACLServiceImplTest { @@ -80,6 +83,8 @@ public class NetworkACLServiceImplTest { @Mock private EntityManager entityManagerMock; @Mock + private VpcDao vpcDaoMock; + @Mock private AccountManager accountManagerMock; @Mock private NetworkACLDao networkAclDaoMock; @@ -99,10 +104,11 @@ public class NetworkACLServiceImplTest { @Mock private UpdateNetworkACLListCmd updateNetworkACLListCmdMock; - private Long networkAclMockId = 1L; + private Long networkAclMockId = 5L; private Long networkOfferingMockId = 2L; private Long networkMockVpcMockId = 3L; private long networkAclListId = 1l; + private static final String SOME_UUID = "someUuid"; @Mock private MoveNetworkAclItemCmd moveNetworkAclItemCmdMock; @@ -121,6 +127,12 @@ public class NetworkACLServiceImplTest { @Mock private CallContext callContextMock; + @Mock + private VpcVO vpcVOMock; + + @Mock + private Account accountMock; + private MockedStatic callContextMocked; @Before @@ -178,9 +190,9 @@ public class NetworkACLServiceImplTest { } }); - NetworkACLItem netowkrAclRuleCreated = networkAclServiceImpl.createNetworkACLItem(createNetworkAclCmdMock); + NetworkACLItem networkAclRuleCreated = networkAclServiceImpl.createNetworkACLItem(createNetworkAclCmdMock); - Assert.assertEquals(number == null ? 6 : number, netowkrAclRuleCreated.getNumber()); + Assert.assertEquals(number == null ? 6 : number, networkAclRuleCreated.getNumber()); InOrder inOrder = Mockito.inOrder( networkAclServiceImpl, networkAclManagerMock, networkAclItemDaoMock); inOrder.verify(networkAclServiceImpl).createAclListIfNeeded(createNetworkAclCmdMock); @@ -392,13 +404,13 @@ public class NetworkACLServiceImplTest { @Test(expected = InvalidParameterValueException.class) public void validateNetworkAclTestAclNotDefaulWithoutVpc() { Mockito.when(networkAclMock.getId()).thenReturn(3L); - Mockito.doReturn(null).when(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); + Mockito.doReturn(null).when(vpcDaoMock).findById(networkMockVpcMockId); networkAclServiceImpl.validateNetworkAcl(networkAclMock); } @Test - public void validateNetworkAclTestAclNotDefaulWithVpc() { + public void validateNetworkAclTestAclNotDefaultWithVpc() { CallContext callContextMock = Mockito.mock(CallContext.class); Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount(); @@ -407,12 +419,11 @@ public class NetworkACLServiceImplTest { Mockito.when(networkAclMock.getId()).thenReturn(3L); Mockito.when(networkAclMock.getVpcId()).thenReturn(networkMockVpcMockId); - Mockito.doReturn(Mockito.mock(Vpc.class)).when(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(networkMockVpcMockId); Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); networkAclServiceImpl.validateNetworkAcl(networkAclMock); - Mockito.verify(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); Mockito.verify(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); callContextMocked.verify(() -> CallContext.current()); @@ -702,16 +713,22 @@ public class NetworkACLServiceImplTest { Mockito.doReturn(networkAclItemVoMock).when(networkAclServiceImpl).validateNetworkAclRuleIdAndRetrieveIt(updateNetworkACLItemCmdMock); Mockito.doReturn(networkAclMock).when(networkAclManagerMock).getNetworkACL(networkAclMockId); Mockito.doNothing().when(networkAclServiceImpl).validateNetworkAcl(Mockito.eq(networkAclMock)); + Mockito.doNothing().when(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(Mockito.any(NetworkACL.class), Mockito.any(Account.class), Mockito.anyString()); Mockito.doNothing().when(networkAclServiceImpl).transferDataToNetworkAclRulePojo(Mockito.eq(updateNetworkACLItemCmdMock), Mockito.eq(networkAclItemVoMock), Mockito.eq(networkAclMock)); Mockito.doNothing().when(networkAclServiceImpl).validateNetworkACLItem(networkAclItemVoMock); Mockito.doReturn(networkAclItemVoMock).when(networkAclManagerMock).updateNetworkACLItem(networkAclItemVoMock); + CallContext callContextMock = Mockito.mock(CallContext.class); + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.when(CallContext.current()).thenReturn(callContextMock); + networkAclServiceImpl.updateNetworkACLItem(updateNetworkACLItemCmdMock); InOrder inOrder = Mockito.inOrder(networkAclServiceImpl, networkAclManagerMock); inOrder.verify(networkAclServiceImpl).validateNetworkAclRuleIdAndRetrieveIt(updateNetworkACLItemCmdMock); inOrder.verify(networkAclManagerMock).getNetworkACL(networkAclMockId); inOrder.verify(networkAclServiceImpl).validateNetworkAcl(networkAclMock); + inOrder.verify(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(networkAclMock, accountMock, "Only Root Admins can update global ACLs."); inOrder.verify(networkAclServiceImpl).transferDataToNetworkAclRulePojo(Mockito.eq(updateNetworkACLItemCmdMock), Mockito.eq(networkAclItemVoMock), Mockito.eq(networkAclMock)); inOrder.verify(networkAclServiceImpl).validateNetworkACLItem(networkAclItemVoMock); inOrder.verify(networkAclManagerMock).updateNetworkACLItem(networkAclItemVoMock); @@ -865,13 +882,18 @@ public class NetworkACLServiceImplTest { Mockito.when(updateNetworkACLListCmdMock.getCustomId()).thenReturn(customId); Mockito.when(updateNetworkACLListCmdMock.getId()).thenReturn(networkAclListId); Mockito.when(updateNetworkACLListCmdMock.getDisplay()).thenReturn(false); + Mockito.when(networkACLVOMock.getVpcId()).thenReturn(networkMockVpcMockId); + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(networkMockVpcMockId); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), + Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); + networkAclServiceImpl.updateNetworkACL(updateNetworkACLListCmdMock); - InOrder inOrder = Mockito.inOrder(networkAclDaoMock, entityManagerMock, entityManagerMock, accountManagerMock, networkACLVOMock); + InOrder inOrder = Mockito.inOrder(networkAclDaoMock, vpcDaoMock, accountManagerMock, networkACLVOMock); inOrder.verify(networkAclDaoMock).findById(networkAclListId); - inOrder.verify(entityManagerMock).findById(Mockito.eq(Vpc.class), Mockito.anyLong()); + inOrder.verify(vpcDaoMock).findById(Mockito.anyLong()); inOrder.verify(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), nullable(Vpc.class)); @@ -1063,13 +1085,17 @@ public class NetworkACLServiceImplTest { Mockito.when(updateNetworkACLListCmdMock.getCustomId()).thenReturn(null); Mockito.when(updateNetworkACLListCmdMock.getId()).thenReturn(networkAclListId); Mockito.when(updateNetworkACLListCmdMock.getDisplay()).thenReturn(null); + Mockito.when(networkACLVOMock.getVpcId()).thenReturn(networkMockVpcMockId); + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(networkMockVpcMockId); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), + Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); networkAclServiceImpl.updateNetworkACL(updateNetworkACLListCmdMock); - InOrder inOrder = Mockito.inOrder(networkAclDaoMock, entityManagerMock, accountManagerMock, networkACLVOMock); + InOrder inOrder = Mockito.inOrder(networkAclDaoMock, vpcDaoMock, accountManagerMock, networkACLVOMock); inOrder.verify(networkAclDaoMock).findById(networkAclListId); - inOrder.verify(entityManagerMock).findById(eq(Vpc.class), Mockito.anyLong()); + inOrder.verify(vpcDaoMock).findById(Mockito.anyLong()); inOrder.verify(accountManagerMock).checkAccess(any(Account.class), isNull(), eq(true), nullable(Vpc.class)); Mockito.verify(networkACLVOMock, Mockito.times(0)).setName(null); @@ -1086,21 +1112,18 @@ public class NetworkACLServiceImplTest { Mockito.when(nextAclRuleMock.getAclId()).thenReturn(networkAclMockId); Mockito.when(previousAclRuleMock.getAclId()).thenReturn(networkAclMockId); - Mockito.doReturn(networkAclMock).when(networkAclDaoMock).findById(networkAclMockId); - Mockito.doReturn(Mockito.mock(Vpc.class)).when(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); - CallContext callContextMock = Mockito.mock(CallContext.class); Mockito.doReturn(Mockito.mock(Account.class)).when(callContextMock).getCallingAccount(); + Mockito.doReturn(networkAclMock).when(networkAclDaoMock).findById(networkAclMockId); Mockito.when(CallContext.current()).thenReturn(callContextMock); - Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); + Mockito.doNothing().when(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(Mockito.any(NetworkACL.class), Mockito.any(Account.class), Mockito.anyString()); networkAclServiceImpl.validateMoveAclRulesData(aclRuleBeingMovedMock, previousAclRuleMock, nextAclRuleMock); Mockito.verify(networkAclDaoMock).findById(networkAclMockId); - Mockito.verify(entityManagerMock).findById(Vpc.class, networkMockVpcMockId); - Mockito.verify(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.isNull(AccessType.class), Mockito.eq(true), Mockito.any(Vpc.class)); + Mockito.verify(networkAclServiceImpl).validateGlobalAclPermissionAndAclAssociatedToVpc(Mockito.any(NetworkACL.class), Mockito.any(Account.class), Mockito.anyString()); } @Test @@ -1359,9 +1382,41 @@ public class NetworkACLServiceImplTest { ArrayList allAclRules = new ArrayList<>(); allAclRules.add(networkAclItemVoMock); - Mockito.doReturn("someUuid").when(networkAclItemVoMock).getUuid(); + Mockito.doReturn(SOME_UUID).when(networkAclItemVoMock).getUuid(); networkAclServiceImpl.validateAclConsistency(moveNetworkAclItemCmdMock, networkACLVOMock, allAclRules); Mockito.verify(moveNetworkAclItemCmdMock, Mockito.times(1)).getAclConsistencyHash(); } + + @Test + public void checkGlobalAclPermissionTestGlobalAclWithRootAccountShouldWork() { + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(true).when(networkAclServiceImpl).isGlobalAcl(Mockito.anyLong()); + + networkAclServiceImpl.checkGlobalAclPermission(networkMockVpcMockId, accountMock, "exception"); + } + + @Test(expected = PermissionDeniedException.class) + public void checkGlobalAclPermissionTestGlobalAclWithNonRootAccountShouldThrow() { + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + Mockito.doReturn(true).when(networkAclServiceImpl).isGlobalAcl(Mockito.anyLong()); + + networkAclServiceImpl.checkGlobalAclPermission(networkMockVpcMockId, accountMock, "exception"); + } + + @Test + public void validateAclAssociatedToVpcTestNonNullVpcShouldCheckAccess() { + Mockito.doReturn(vpcVOMock).when(vpcDaoMock).findById(Mockito.anyLong()); + + networkAclServiceImpl.validateAclAssociatedToVpc(networkMockVpcMockId, accountMock, SOME_UUID); + + Mockito.verify(accountManagerMock, Mockito.times(1)).checkAccess(Mockito.any(Account.class), isNull(), Mockito.anyBoolean(), Mockito.any(Vpc.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateAclAssociatedToVpcTestNullVpcShouldThrowInvalidParameterValueException() { + Mockito.doReturn(null).when(vpcDaoMock).findById(Mockito.anyLong()); + + networkAclServiceImpl.validateAclAssociatedToVpc(networkMockVpcMockId, accountMock, SOME_UUID); + } } diff --git a/test/integration/smoke/test_global_acls.py b/test/integration/smoke/test_global_acls.py new file mode 100644 index 00000000000..47db858c121 --- /dev/null +++ b/test/integration/smoke/test_global_acls.py @@ -0,0 +1,245 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.lib.utils import cleanup_resources +from marvin.lib.base import (Network, NetworkACLList, NetworkOffering, VpcOffering, VPC, NetworkACL) +from marvin.lib.common import (get_domain, get_zone) +from nose.plugins.attrib import attr +from marvin.cloudstackException import CloudstackAPIException + + +class Services: + """Test Global ACLs + """ + + def __init__(self): + self.services = { + "root_domain": { + "name": "ROOT", + }, + "domain": { + "name": "Domain", + }, + "user": { + "username": "user", + "roletype": 0, + }, + "domain_admin": { + "username": "Domain admin", + "roletype": 2, + }, + "root_admin": { + "username": "Root admin", + "roletype": 1, + }, + "vpc": { + "name": "vpc-networkacl", + "displaytext": "vpc-networkacl", + "cidr": "10.1.1.0/24", + }, + "vpcnetwork": { + "name": "vpcnetwork", + "displaytext": "vpcnetwork", + }, + "rule": { + "protocol": "all", + "traffictype": "ingress", + } + } + + +class TestGlobalACLs(cloudstackTestCase): + + @classmethod + def setUpClass(cls): + cls.testClient = super(TestGlobalACLs, cls).getClsTestClient() + cls.apiclient = cls.testClient.getApiClient() + + cls.services = Services().services + cls.domain = get_domain(cls.apiclient) + cls.zone = get_zone(cls.apiclient, cls.testClient.getZoneForTests()) + return + + def setUp(self): + self.user_apiclient = self.testClient.getUserApiClient(self.services["user"]["username"], + self.services["domain"]["name"], + self.services["user"]["roletype"]) + + self.domain_admin_apiclient = self.testClient.getUserApiClient(self.services["domain_admin"]["username"], + self.services["domain"]["name"], + self.services["domain_admin"]["roletype"]) + + self.admin_apiclient = self.testClient.getUserApiClient(self.services["root_admin"]["username"], + self.services["root_domain"]["name"], + self.services["root_admin"]["roletype"]) + + self.cleanup = [] + return + + def tearDown(self): + super(TestGlobalACLs, self).tearDown() + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_create_global_acl(self): + """ Test create global ACL as a normal user, domain admin and root admin users. + """ + + self.debug("Creating ACL list as a normal user, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admin can create global ACLs.", + NetworkACLList.create, apiclient=self.user_apiclient, services={}, + name="acl", description="acl") + + self.debug("Creating ACL list as a domain admin, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admin can create global ACLs.", + NetworkACLList.create, apiclient=self.domain_admin_apiclient, services={}, + name="acl", description="acl") + + self.debug("Creating ACL list as a root admin, should work.") + acl = NetworkACLList.create(apiclient=self.admin_apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + self.assertIsNotNone(acl, "A root admin user should be able to create a global ACL.") + + return + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_replace_acl_of_network(self): + """ Test to replace ACL of a VPC as a normal user, domain admin and root admin users. + """ + # Get network offering + networkOffering = NetworkOffering.list(self.apiclient, name="DefaultIsolatedNetworkOfferingForVpcNetworks") + self.assertTrue(networkOffering is not None and len(networkOffering) > 0, "No VPC network offering") + + # Getting VPC offering + vpcOffering = VpcOffering.list(self.apiclient, name="Default VPC offering") + self.assertTrue(vpcOffering is not None and len(vpcOffering) > 0, "No VPC offerings found") + + # Creating VPC + vpc = VPC.create( + apiclient=self.apiclient, + services=self.services["vpc"], + networkDomain="vpc.networkacl", + vpcofferingid=vpcOffering[0].id, + zoneid=self.zone.id, + domainid=self.domain.id + ) + self.cleanup.append(vpc) + self.assertTrue(vpc is not None, "VPC creation failed") + + # Creating ACL list + acl = NetworkACLList.create(apiclient=self.apiclient, services={}, name="acl", description="acl") + + # Creating tier on VPC with ACL list + network = Network.create( + apiclient=self.apiclient, + services=self.services["vpcnetwork"], + accountid="Admin", + domainid=self.domain.id, + networkofferingid=networkOffering[0].id, + zoneid=self.zone.id, + vpcid=vpc.id, + aclid=acl.id, + gateway="10.1.1.1", + netmask="255.255.255.192" + ) + self.cleanup.append(network) + + # User should be able to replace ACL + network.replaceACLList(apiclient=self.user_apiclient, aclid=acl.id) + # Domain Admin should be able to replace ACL + network.replaceACLList(apiclient=self.domain_admin_apiclient, aclid=acl.id) + # Admin should be able to replace ACL + network.replaceACLList(apiclient=self.admin_apiclient, aclid=acl.id) + + return + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_create_acl_rule(self): + """ Test to create ACL rule as a normal user, domain admin and root admin users. + """ + # Creating ACL list + acl = NetworkACLList.create(apiclient=self.admin_apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + + self.debug("Creating ACL rule as a user, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admins can create rules for a global ACL.", + NetworkACL.create, self.user_apiclient, services=self.services["rule"], aclid=acl.id) + self.debug("Creating ACL rule as a domain admin, should raise exception.") + self.assertRaisesRegex(CloudstackAPIException, "Only Root Admins can create rules for a global ACL.", + NetworkACL.create, self.domain_admin_apiclient, services=self.services["rule"], aclid=acl.id) + self.debug("Creating ACL rule as a root admin, should work.") + acl_rule = NetworkACL.create(self.admin_apiclient, services=self.services["rule"], aclid=acl.id) + self.cleanup.append(acl_rule) + + return + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_delete_acl_rule(self): + """ Test to delete ACL rule as a normal user, domain admin and root admin users. + """ + # Creating ACL list + acl = NetworkACLList.create(apiclient=self.apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + + # Creating ACL rule + acl_rule = NetworkACL.create(self.apiclient, services=self.services["rule"], aclid=acl.id) + self.cleanup.append(acl_rule) + + self.debug("Deleting ACL rule as a user, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACL rules.", + NetworkACL.delete, acl_rule, self.user_apiclient) + self.debug("Deleting ACL rule as a domain admin, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACL rules.", + NetworkACL.delete, acl_rule, self.domain_admin_apiclient) + + self.debug("Deleting ACL rule as a root admin, should work.") + NetworkACL.delete(acl_rule, self.admin_apiclient) + self.cleanup.remove(acl_rule) + + # Verify if the number of ACL rules is equal to four, i.e. the number of rules + # for the default ACLs `default_allow` (2 rules) and `default_deny` (2 rules) ACLs + number_of_acl_rules = acl_rule.list(apiclient=self.admin_apiclient) + self.assertEqual(len(number_of_acl_rules), 4) + + return + + + @attr(tags=["advanced", "basic"], required_hardware="false") + def test_delete_global_acl(self): + """ Test delete global ACL as a normal user, domain admin and root admin users. + """ + + # Creating ACL list. Not adding to cleanup as it will be deleted in this method + acl = NetworkACLList.create(apiclient=self.apiclient, services={}, name="acl", description="acl") + self.cleanup.append(acl) + + self.debug("Deleting ACL list as a normal user, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACLs.", + NetworkACLList.delete, acl, apiclient=self.user_apiclient) + + self.debug("Deleting ACL list as a domain admin, should raise exception.") + self.assertRaisesRegex(Exception, "Only Root Admin can delete global ACLs.", + NetworkACLList.delete, acl, apiclient=self.domain_admin_apiclient) + + self.debug("Deleting ACL list as a root admin, should work.") + acl.delete(apiclient=self.admin_apiclient) + self.cleanup.remove(acl) + + # Verify if number of ACLs is equal to two, i.e. the number of default ACLs `default_allow` and `default_deny` + number_of_acls = NetworkACLList.list(apiclient=self.admin_apiclient) + self.assertEqual(len(number_of_acls), 2) + + return From 37a2350159dafeb4ab934cd1560f6a5a07346603 Mon Sep 17 00:00:00 2001 From: Fabricio Duarte Date: Fri, 1 Dec 2023 04:40:43 -0300 Subject: [PATCH 14/50] Fix upload of volumes, templates and ISOs through HTTP (#8264) When HTTP is used to upload volumes, templates and ISOs (configuration use.https.to.upload is set to false), an error occurs during the validation of the request's signature, preventing uploads from finishing successfully. This happens because an URL in the format String fullUrl = "https://" + hostname + "/upload/" + uuid is always created for the validation whether HTTP or HTTPS was being used. This PR addresses this issue, allowing users to upload volumes, templates and ISOs through HTTP. --- .../resource/NfsSecondaryStorageResource.java | 56 ++++++++-- .../NfsSecondaryStorageResourceTest.java | 101 ++++++++++++++++++ .../java/com/cloud/utils/net/NetUtils.java | 1 + 3 files changed, 149 insertions(+), 9 deletions(-) diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 5d76393d81e..cc17e48fe07 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -85,6 +85,7 @@ import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; @@ -201,6 +202,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S private static final String POST_UPLOAD_KEY_LOCATION = "/etc/cloudstack/agent/ms-psk"; private static final String ORIGINAL_FILE_EXTENSION = ".orig"; + private static final String USE_HTTPS_TO_UPLOAD = "useHttpsToUpload"; + private static final Map updatableConfigData = Maps.newHashMap(); static { @@ -3537,7 +3540,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return null; } - private String getPostUploadPSK() { + protected String getPostUploadPSK() { if (_ssvmPSK == null) { try { _ssvmPSK = FileUtils.readFileToString(new File(POST_UPLOAD_KEY_LOCATION), "utf-8"); @@ -3573,14 +3576,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S throw new InvalidParameterValueException("content length is not set in the request or has invalid value."); } - //validate signature - String fullUrl = "https://" + hostname + "/upload/" + uuid; - String computedSignature = EncryptionUtil.generateSignature(metadata + fullUrl + timeout, getPostUploadPSK()); - boolean isSignatureValid = computedSignature.equals(signature); - if (!isSignatureValid) { - updateStateMapWithError(uuid, "signature validation failed."); - throw new InvalidParameterValueException("signature validation failed."); - } + validatePostUploadRequestSignature(signature, hostname, uuid, metadata, timeout); //validate timeout DateTime timeoutDateTime = DateTime.parse(timeout, ISODateTimeFormat.dateTime()); @@ -3590,6 +3586,48 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + /** + * Validates whether the provided signature matches the signature generated from the other parameters; + * throws an InvalidParameterValueException if it does not. + */ + protected void validatePostUploadRequestSignature(String signature, String hostname, String uuid, String metadata, String timeout) { + s_logger.trace(String.format("Validating signature [%s] for post upload request [%s].", signature, uuid)); + String protocol = getUploadProtocol(); + String fullUrl = String.format("%s://%s/upload/%s", protocol, hostname, uuid); + String data = String.format("%s%s%s", metadata, fullUrl, timeout); + + String computedSignature = EncryptionUtil.generateSignature(data, getPostUploadPSK()); + s_logger.debug(String.format("Computed signature for post upload request [%s] is [%s].", uuid, computedSignature)); + + boolean isSignatureValid = computedSignature.equals(signature); + if (!isSignatureValid) { + s_logger.debug(String.format("Signature for post upload request [%s] is invalid.", uuid)); + String errorMsg = "signature validation failed."; + updateStateMapWithError(uuid, errorMsg); + throw new InvalidParameterValueException(errorMsg); + } + s_logger.debug(String.format("Signature for post upload request [%s] is valid.", uuid)); + } + + /** + * Returns the protocol used for uploads as a string. + */ + protected String getUploadProtocol() { + if (useHttpsToUpload()) { + s_logger.debug(String.format("Param [%s] is set to true; therefore, HTTPS is being used.", USE_HTTPS_TO_UPLOAD)); + return NetUtils.HTTPS_PROTO; + } + s_logger.debug(String.format("Param [%s] is set to false; therefore, HTTP is being used.", USE_HTTPS_TO_UPLOAD)); + return NetUtils.HTTP_PROTO; + } + + /** + * Retrieves the value of "useHttpsToUpload" from the params as a boolean + */ + protected boolean useHttpsToUpload() { + return BooleanUtils.toBoolean((String) _params.get(USE_HTTPS_TO_UPLOAD)); + } + private TemplateOrVolumePostUploadCommand getTemplateOrVolumePostUploadCmd(String metadata) { TemplateOrVolumePostUploadCommand cmd = null; try { diff --git a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index cd6444a8b4b..72b9f5abe19 100644 --- a/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/src/test/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -28,6 +28,9 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Stream; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.utils.EncryptionUtil; +import com.cloud.utils.net.NetUtils; import org.apache.cloudstack.storage.command.DeleteCommand; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; @@ -51,6 +54,22 @@ public class NfsSecondaryStorageResourceTest { @Spy private NfsSecondaryStorageResource resource; + private static final String HOSTNAME = "hostname"; + + private static final String UUID = "uuid"; + + private static final String METADATA = "metadata"; + + private static final String TIMEOUT = "timeout"; + + private static final String PSK = "6HyGMx9Vat7rZw1pMZrM4OlD4FFwLUPznTsFqVFSOIvk0mAWMRCVZ6UCq42gZvhp"; + + private static final String PROTOCOL = NetUtils.HTTP_PROTO; + + private static final String EXPECTED_SIGNATURE = "expectedSignature"; + + private static final String COMPUTED_SIGNATURE = "computedSignature"; + @Test public void testSwiftWriteMetadataFile() throws Exception { String metaFileName = "test_metadata_file"; @@ -141,4 +160,86 @@ public class NfsSecondaryStorageResourceTest { Assert.assertEquals(dir + File.separator + fileName + ".ovf", result.get(1)); } } + + private void prepareForValidatePostUploadRequestSignatureTests(MockedStatic encryptionUtilMock) { + Mockito.doReturn(PROTOCOL).when(resource).getUploadProtocol(); + Mockito.doReturn(PSK).when(resource).getPostUploadPSK(); + encryptionUtilMock.when(() -> EncryptionUtil.generateSignature(Mockito.anyString(), Mockito.anyString())).thenReturn(COMPUTED_SIGNATURE); + String fullUrl = String.format("%s://%s/upload/%s", PROTOCOL, HOSTNAME, UUID); + String data = String.format("%s%s%s", METADATA, fullUrl, TIMEOUT); + encryptionUtilMock.when(() -> EncryptionUtil.generateSignature(data, PSK)).thenReturn(EXPECTED_SIGNATURE); + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenProtocolDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + Mockito.doReturn(NetUtils.HTTPS_PROTO).when(resource).getUploadProtocol(); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, METADATA, TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenHostnameDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, "test", UUID, METADATA, TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenUuidDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, "test", METADATA, TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenMetadataDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, "test", TIMEOUT); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void validatePostUploadRequestSignatureTestThrowExceptionWhenTimeoutDiffers() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, METADATA, "test"); + } + } + + @Test + public void validatePostUploadRequestSignatureTestSuccessWhenDataIsTheSame() { + try (MockedStatic encryptionUtilMock = Mockito.mockStatic(EncryptionUtil.class)) { + prepareForValidatePostUploadRequestSignatureTests(encryptionUtilMock); + + resource.validatePostUploadRequestSignature(EXPECTED_SIGNATURE, HOSTNAME, UUID, METADATA, TIMEOUT); + } + } + + @Test + public void getUploadProtocolTestReturnHttpsWhenUseHttpsToUploadIsTrue() { + Mockito.doReturn(true).when(resource).useHttpsToUpload(); + + String result = resource.getUploadProtocol(); + + Assert.assertEquals(NetUtils.HTTPS_PROTO, result); + } + + @Test + public void getUploadProtocolTestReturnHttpWhenUseHttpsToUploadIsFalse() { + Mockito.doReturn(false).when(resource).useHttpsToUpload(); + + String result = resource.getUploadProtocol(); + + Assert.assertEquals(NetUtils.HTTP_PROTO, result); + } } diff --git a/utils/src/main/java/com/cloud/utils/net/NetUtils.java b/utils/src/main/java/com/cloud/utils/net/NetUtils.java index 137cebb516a..018912a2140 100644 --- a/utils/src/main/java/com/cloud/utils/net/NetUtils.java +++ b/utils/src/main/java/com/cloud/utils/net/NetUtils.java @@ -80,6 +80,7 @@ public class NetUtils { public static final String ICMP6_PROTO = "icmp6"; public final static String ALL_PROTO = "all"; public final static String HTTP_PROTO = "http"; + public final static String HTTPS_PROTO = "https"; public final static String SSL_PROTO = "ssl"; public final static String ALL_IP4_CIDRS = "0.0.0.0/0"; From 7d6dc41f99b546f53ded4834cb6f796d69ce5e04 Mon Sep 17 00:00:00 2001 From: Oleg Chuev <91337563+OlegChuev@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:11:11 +0200 Subject: [PATCH 15/50] UI: Removed redundant IP Address Column when create Port forwarding rules (#8275) --- ui/src/views/network/PortForwarding.vue | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ui/src/views/network/PortForwarding.vue b/ui/src/views/network/PortForwarding.vue index edbf49a87e9..fc773c0aea5 100644 --- a/ui/src/views/network/PortForwarding.vue +++ b/ui/src/views/network/PortForwarding.vue @@ -423,11 +423,6 @@ export default { title: this.$t('label.displayname'), dataIndex: 'displayname' }, - { - title: this.$t('label.ip'), - dataIndex: 'ip', - width: 100 - }, { title: this.$t('label.account'), dataIndex: 'account' From 724394682c73d3aaa7991ab899c97c2c3dcbbb63 Mon Sep 17 00:00:00 2001 From: dahn Date: Fri, 1 Dec 2023 09:51:54 +0100 Subject: [PATCH 16/50] server: Initial new vpnuser state (#8268) --- .../apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java | 3 +++ .../org/apache/cloudstack/api/response/VpnUsersResponse.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java index 9c9f1c0f50f..f3b452008c9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpn/AddVpnUserCmd.java @@ -125,6 +125,9 @@ public class AddVpnUserCmd extends BaseAsyncCreateCmd { vpnResponse.setId(vpnUser.getUuid()); vpnResponse.setUserName(vpnUser.getUsername()); vpnResponse.setAccountName(account.getAccountName()); + // re-retrieve the vpnuser, as the call to `applyVpnUsers` might have changed the state + vpnUser = _entityMgr.findById(VpnUser.class, getEntityId()); + vpnResponse.setState(vpnUser.getState().toString()); Domain domain = _entityMgr.findById(Domain.class, account.getDomainId()); if (domain != null) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java index 3a0e84285aa..d3e4d941678 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpnUsersResponse.java @@ -57,7 +57,7 @@ public class VpnUsersResponse extends BaseResponse implements ControlledEntityRe private String projectName; @SerializedName(ApiConstants.STATE) - @Param(description = "the state of the Vpn User") + @Param(description = "the state of the Vpn User, can be 'Add', 'Revoke' or 'Active'.") private String state; public void setId(String id) { From 5651eab49cf3eab58b8d3d14dcc067be5d0e676c Mon Sep 17 00:00:00 2001 From: kishankavala Date: Fri, 1 Dec 2023 17:51:00 +0530 Subject: [PATCH 17/50] ObjectStore Framework with MinIO and Simulator plugins (#7752) This PR adds Object Storage feature to CloudStack. FS: https://cwiki.apache.org/confluence/display/CLOUDSTACK/%5BDRAFT%5D+CloudStack+Object+Store --- .github/workflows/ci.yml | 1 + .../main/java/com/cloud/event/EventTypes.java | 22 + .../java/com/cloud/server/ResourceTag.java | 3 +- .../java/com/cloud/storage/DataStoreRole.java | 4 +- .../com/cloud/storage/StorageService.java | 8 + .../annotation/AnnotationService.java | 3 +- .../api/ApiCommandResourceType.java | 4 +- .../apache/cloudstack/api/ApiConstants.java | 7 + .../org/apache/cloudstack/api/BaseCmd.java | 4 + .../cloudstack/api/ResponseGenerator.java | 8 + .../storage/AddObjectStoragePoolCmd.java | 132 +++ .../storage/DeleteObjectStoragePoolCmd.java | 69 ++ .../storage/ListObjectStoragePoolsCmd.java | 79 ++ .../storage/UpdateObjectStoragePoolCmd.java | 93 +++ .../command/user/bucket/CreateBucketCmd.java | 202 +++++ .../command/user/bucket/DeleteBucketCmd.java | 94 +++ .../command/user/bucket/ListBucketsCmd.java | 100 +++ .../command/user/bucket/UpdateBucketCmd.java | 132 +++ .../api/response/BucketResponse.java | 293 +++++++ .../api/response/ObjectStoreResponse.java | 106 +++ .../apache/cloudstack/query/QueryService.java | 8 + .../cloudstack/storage/object/Bucket.java | 64 ++ .../storage/object/BucketApiService.java | 54 ++ .../storage/object/ObjectStore.java | 40 + .../apache/cloudstack/usage/UsageTypes.java | 2 + .../storage/AddObjectStoragePoolCmdTest.java | 97 +++ .../DeleteObjectStoragePoolCmdTest.java | 59 ++ .../UpdateObjectStoragePoolCmdTest.java | 85 ++ client/pom.xml | 15 + .../subsystem/api/storage/BucketInfo.java | 30 + .../api/storage/DataStoreProvider.java | 2 +- .../api/storage/DataStoreProviderManager.java | 2 + .../api/storage/ObjectStorageService.java | 22 + .../api/storage/ObjectStoreProvider.java | 23 + .../storage/object/ObjectStoreEntity.java | 48 ++ engine/pom.xml | 1 + .../main/java/com/cloud/storage/BucketVO.java | 257 ++++++ .../java/com/cloud/storage/dao/BucketDao.java | 30 + .../com/cloud/storage/dao/BucketDaoImpl.java | 84 ++ .../com/cloud/usage/BucketStatisticsVO.java | 74 ++ .../cloud/usage/dao/BucketStatisticsDao.java | 30 + .../usage/dao/BucketStatisticsDaoImpl.java | 67 ++ .../java/com/cloud/usage/dao/UsageDao.java | 7 + .../com/cloud/usage/dao/UsageDaoImpl.java | 71 ++ .../storage/datastore/db/ObjectStoreDao.java | 42 + .../datastore/db/ObjectStoreDaoImpl.java | 162 ++++ .../datastore/db/ObjectStoreDetailVO.java | 77 ++ .../datastore/db/ObjectStoreDetailsDao.java | 31 + .../db/ObjectStoreDetailsDaoImpl.java | 104 +++ .../storage/datastore/db/ObjectStoreVO.java | 143 ++++ ...spring-engine-schema-core-daos-context.xml | 2 + .../META-INF/db/schema-41810to41900.sql | 66 ++ .../src/test/resources/storageContext.xml | 4 + engine/storage/object/pom.xml | 37 + .../storage/object/BucketObject.java | 196 +++++ .../object/ObjectStorageServiceImpl.java | 28 + .../ObjectStoreProviderManagerImpl.java | 111 +++ .../storage/object/store/ObjectStoreImpl.java | 184 ++++ .../store/lifecycle/ObjectStoreLifeCycle.java | 24 + ...ing-engine-storage-object-core-context.xml | 36 + .../ObjectStoreProviderManagerImplTest.java | 117 +++ .../object/src/test/resource/testContext.xml | 79 ++ .../org.mockito.plugins.MockMaker | 1 + .../datastore/DataStoreManagerImpl.java | 7 + .../DataStoreProviderManagerImpl.java | 27 +- .../object/BaseObjectStoreDriverImpl.java | 81 ++ .../storage/object/ObjectStoreDriver.java | 59 ++ .../object/datastore/ObjectStoreHelper.java | 72 ++ .../datastore/ObjectStoreProviderManager.java | 38 + .../cloudstack/quota/constant/QuotaTypes.java | 1 + .../metrics/MetricsServiceImpl.java | 5 + .../response/InfrastructureResponse.java | 8 + plugins/pom.xml | 26 + .../shutdown/ShutdownManagerImplTest.java | 14 + plugins/storage/object/minio/pom.xml | 57 ++ .../driver/MinIOObjectStoreDriverImpl.java | 404 +++++++++ .../MinIOObjectStoreLifeCycleImpl.java | 129 +++ .../MinIOObjectStoreProviderImpl.java | 85 ++ .../storage-object-minio/module.properties | 18 + .../spring-storage-object-minio-context.xml | 31 + .../MinIOObjectStoreDriverImplTest.java | 122 +++ .../MinIOObjectStoreProviderImplTest.java | 50 ++ plugins/storage/object/simulator/pom.xml | 57 ++ .../SimulatorObjectStoreDriverImpl.java | 139 +++ .../SimulatorObjectStoreLifeCycleImpl.java | 120 +++ .../SimulatorObjectStoreProviderImpl.java | 85 ++ .../module.properties | 18 + ...pring-storage-object-simulator-context.xml | 31 + .../SimulatorObjectStoreProviderImplTest.java | 50 ++ .../main/java/com/cloud/api/ApiDBUtils.java | 23 + .../java/com/cloud/api/ApiResponseHelper.java | 52 +- .../com/cloud/api/query/QueryManagerImpl.java | 166 ++++ .../cloud/api/query/ViewResponseHelper.java | 19 + .../cloud/server/ManagementServerImpl.java | 18 + .../com/cloud/storage/StorageManagerImpl.java | 136 +++ .../annotation/AnnotationManagerImpl.java | 1 + .../storage/object/BucketApiServiceImpl.java | 304 +++++++ .../spring-server-core-managers-context.xml | 1 + .../cloud/api/query/QueryManagerImplTest.java | 52 ++ test/integration/smoke/test_bucket.py | 111 +++ test/integration/smoke/test_object_stores.py | 108 +++ tools/apidoc/gen_toc.py | 10 +- tools/marvin/marvin/lib/base.py | 79 ++ ui/package-lock.json | 790 ++++++++++++++---- ui/package.json | 1 + ui/public/locales/en.json | 38 + ui/src/components/KeyValuePairInput.vue | 91 ++ ui/src/components/view/AnnotationsTab.vue | 1 + ui/src/components/view/ListView.vue | 7 +- ui/src/components/view/ObjectStoreBrowser.vue | 541 ++++++++++++ ui/src/config/section/infra.js | 2 + ui/src/config/section/infra/objectStorages.js | 74 ++ ui/src/config/section/storage.js | 129 ++- ui/src/core/lazy_lib/icons_use.js | 2 + ui/src/views/infra/AddObjectStorage.vue | 171 ++++ ui/src/views/infra/InfraSummary.vue | 2 +- ui/src/views/storage/CreateBucket.vue | 196 +++++ ui/src/views/storage/UpdateBucket.vue | 180 ++++ .../com/cloud/usage/UsageManagerImpl.java | 112 ++- .../cloud/usage/parser/BucketUsageParser.java | 78 ++ 120 files changed, 8985 insertions(+), 224 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java create mode 100644 api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java create mode 100644 api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java create mode 100644 api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java create mode 100644 engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java create mode 100644 engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java create mode 100644 engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java create mode 100644 engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java create mode 100644 engine/schema/src/main/java/com/cloud/storage/BucketVO.java create mode 100644 engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java create mode 100644 engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java create mode 100644 engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java create mode 100644 engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java create mode 100644 engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java create mode 100644 engine/storage/object/pom.xml create mode 100644 engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java create mode 100644 engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java create mode 100644 engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java create mode 100644 engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java create mode 100644 engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java create mode 100644 engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml create mode 100644 engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java create mode 100644 engine/storage/object/src/test/resource/testContext.xml create mode 100644 engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker create mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java create mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java create mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java create mode 100644 engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java create mode 100644 plugins/storage/object/minio/pom.xml create mode 100644 plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java create mode 100644 plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java create mode 100644 plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java create mode 100644 plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties create mode 100644 plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml create mode 100644 plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java create mode 100644 plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java create mode 100644 plugins/storage/object/simulator/pom.xml create mode 100644 plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java create mode 100644 plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java create mode 100644 plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java create mode 100644 plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties create mode 100644 plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml create mode 100644 plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java create mode 100644 test/integration/smoke/test_bucket.py create mode 100644 test/integration/smoke/test_object_stores.py create mode 100644 ui/src/components/KeyValuePairInput.vue create mode 100644 ui/src/components/view/ObjectStoreBrowser.vue create mode 100644 ui/src/config/section/infra/objectStorages.js create mode 100644 ui/src/views/infra/AddObjectStorage.vue create mode 100644 ui/src/views/storage/CreateBucket.vue create mode 100644 ui/src/views/storage/UpdateBucket.vue create mode 100644 usage/src/main/java/com/cloud/usage/parser/BucketUsageParser.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77c9d59505c..c6edc7bdb20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: smoke/test_nic smoke/test_nic_adapter_type smoke/test_non_contigiousvlan + smoke/test_object_stores smoke/test_outofbandmanagement smoke/test_outofbandmanagement_nestedplugin smoke/test_over_provisioning diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 1ac8d79b9d5..5fce169ffed 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -29,6 +29,8 @@ import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.ha.HAConfig; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.vm.schedule.VMSchedule; @@ -714,6 +716,16 @@ public class EventTypes { // SystemVM public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM"; + // OBJECT STORE + public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE"; + public static final String EVENT_OBJECT_STORE_DELETE = "OBJECT.STORE.DELETE"; + public static final String EVENT_OBJECT_STORE_UPDATE = "OBJECT.STORE.UPDATE"; + + // BUCKETS + public static final String EVENT_BUCKET_CREATE = "BUCKET.CREATE"; + public static final String EVENT_BUCKET_DELETE = "BUCKET.DELETE"; + public static final String EVENT_BUCKET_UPDATE = "BUCKET.UPDATE"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1151,6 +1163,16 @@ public class EventTypes { entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); entityEventDetails.put(EVENT_IMAGE_STORE_OBJECT_DOWNLOAD, ImageStore.class); entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); + + //Object Store + entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class); + entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class); + entityEventDetails.put(EVENT_OBJECT_STORE_DELETE, ObjectStore.class); + + //Buckets + entityEventDetails.put(EVENT_BUCKET_CREATE, Bucket.class); + entityEventDetails.put(EVENT_BUCKET_UPDATE, Bucket.class); + entityEventDetails.put(EVENT_BUCKET_DELETE, Bucket.class); } public static String getEntityForEvent(String eventName) { diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index 2304241f7e5..9bbb5d43eae 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -69,7 +69,8 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false), - Domain(false, false, true); + Domain(false, false, true), + ObjectStore(false, false, true); ResourceObjectType(boolean resourceTagsSupport, boolean resourceMetadataSupport) { diff --git a/api/src/main/java/com/cloud/storage/DataStoreRole.java b/api/src/main/java/com/cloud/storage/DataStoreRole.java index cc20cc0ce96..185e370159c 100644 --- a/api/src/main/java/com/cloud/storage/DataStoreRole.java +++ b/api/src/main/java/com/cloud/storage/DataStoreRole.java @@ -21,7 +21,7 @@ package com.cloud.storage; import com.cloud.utils.exception.CloudRuntimeException; public enum DataStoreRole { - Primary("primary"), Image("image"), ImageCache("imagecache"), Backup("backup"); + Primary("primary"), Image("image"), ImageCache("imagecache"), Backup("backup"), Object("object"); public boolean isImageStore() { return (role.equalsIgnoreCase("image") || role.equalsIgnoreCase("imagecache")) ? true : false; @@ -45,6 +45,8 @@ public enum DataStoreRole { return ImageCache; } else if (role.equalsIgnoreCase("backup")) { return Backup; + } else if (role.equalsIgnoreCase("object")) { + return Object; } else { throw new CloudRuntimeException("can't identify the role"); } diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index bb086ad05cb..3c6c22f0ea2 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -24,9 +24,11 @@ import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaint import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import com.cloud.exception.DiscoveryException; @@ -34,6 +36,7 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.storage.object.ObjectStore; public interface StorageService { /** @@ -109,4 +112,9 @@ public interface StorageService { StoragePool syncStoragePool(SyncStoragePoolCmd cmd); + ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; + + boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); + + ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd); } diff --git a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java index 51c6286a9d5..64bd96d9a9b 100644 --- a/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java +++ b/api/src/main/java/org/apache/cloudstack/annotation/AnnotationService.java @@ -45,7 +45,7 @@ public interface AnnotationService { SERVICE_OFFERING(false), DISK_OFFERING(false), NETWORK_OFFERING(false), ZONE(false), POD(false), CLUSTER(false), HOST(false), DOMAIN(false), PRIMARY_STORAGE(false), SECONDARY_STORAGE(false), VR(false), SYSTEM_VM(false), - AUTOSCALE_VM_GROUP(true), MANAGEMENT_SERVER(false),; + AUTOSCALE_VM_GROUP(true), MANAGEMENT_SERVER(false), OBJECT_STORAGE(false); private final boolean usersAllowed; @@ -78,6 +78,7 @@ public interface AnnotationService { list.add(EntityType.VR); list.add(EntityType.SYSTEM_VM); list.add(EntityType.MANAGEMENT_SERVER); + list.add(EntityType.OBJECT_STORAGE); if (roleType != RoleType.DomainAdmin) { list.add(EntityType.DOMAIN); list.add(EntityType.SERVICE_OFFERING); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 9267ca6fa96..affceb4e3f9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -78,7 +78,9 @@ public enum ApiCommandResourceType { VmSnapshot(com.cloud.vm.snapshot.VMSnapshot.class), Role(org.apache.cloudstack.acl.Role.class), VpnCustomerGateway(com.cloud.network.Site2SiteCustomerGateway.class), - ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class); + ManagementServer(org.apache.cloudstack.management.ManagementServerHost.class), + ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class), + Bucket(org.apache.cloudstack.storage.object.Bucket.class); private final Class clazz; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 4b437516bd8..fa0a24670bd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1049,10 +1049,17 @@ public class ApiConstants { public static final String MTU = "mtu"; public static final String AUTO_ENABLE_KVM_HOST = "autoenablekvmhost"; public static final String LIST_APIS = "listApis"; + public static final String OBJECT_STORAGE_ID = "objectstorageid"; + public static final String VERSIONING = "versioning"; + public static final String OBJECT_LOCKING = "objectlocking"; + public static final String ENCRYPTION = "encryption"; + public static final String QUOTA = "quota"; + public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; public static final String SOURCE_NAT_IP_ID = "sourcenatipaddressid"; public static final String HAS_RULES = "hasrules"; + public static final String OBJECT_STORAGE = "objectstore"; public static final String MANAGEMENT = "management"; public static final String IS_VNF = "isvnf"; diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index 0b80cfc8229..f32922819b0 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -42,6 +42,7 @@ import org.apache.cloudstack.network.element.InternalLoadBalancerElementService; import org.apache.cloudstack.network.lb.ApplicationLoadBalancerService; import org.apache.cloudstack.network.lb.InternalLoadBalancerVMService; import org.apache.cloudstack.query.QueryService; +import org.apache.cloudstack.storage.object.BucketApiService; import org.apache.cloudstack.storage.ImageStoreService; import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.usage.UsageService; @@ -216,6 +217,9 @@ public abstract class BaseCmd { public Ipv6Service ipv6Service; @Inject public VnfTemplateManager vnfTemplateManager; + @Inject + public BucketApiService _bucketService; + public abstract void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 030f70805d2..944bb2194a5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants.HostDetails; @@ -37,6 +38,7 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.CapacityResponse; import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ConditionResponse; @@ -82,6 +84,7 @@ import org.apache.cloudstack.api.response.NetworkPermissionsResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.NicResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.OvsProviderResponse; import org.apache.cloudstack.api.response.PhysicalNetworkResponse; import org.apache.cloudstack.api.response.PodResponse; @@ -145,6 +148,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import com.cloud.capacity.Capacity; @@ -533,4 +537,8 @@ public interface ResponseGenerator { FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl); IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp); + + ObjectStoreResponse createObjectStoreResponse(ObjectStore os); + + BucketResponse createBucketResponse(Bucket bucket); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java new file mode 100644 index 00000000000..a538962e076 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.storage.object.ObjectStore; +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.log4j.Logger; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +@APICommand(name = "addObjectStoragePool", description = "Adds a object storage pool", responseObject = ObjectStoreResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class AddObjectStoragePoolCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(AddObjectStoragePoolCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "the name for the object store") + private String name; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, length = 2048, required = true, description = "the URL for the object store") + private String url; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, required = true, description = "the object store provider name") + private String providerName; + + @Parameter(name = ApiConstants.DETAILS, + type = CommandType.MAP, + description = "the details for the object store. Example: details[0].key=accesskey&details[0].value=s389ddssaa&details[1].key=secretkey&details[1].value=8dshfsss") + private Map details; + + @Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "the tags for the storage pool") + private String tags; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getUrl() { + return url; + } + + public String getName() { + return name; + } + + public Map getDetails() { + Map detailsMap = null; + if (details != null && !details.isEmpty()) { + detailsMap = new HashMap(); + Collection props = details.values(); + Iterator iter = props.iterator(); + while (iter.hasNext()) { + HashMap detail = (HashMap)iter.next(); + String key = detail.get(ApiConstants.KEY); + String value = detail.get(ApiConstants.VALUE); + detailsMap.put(key, value); + } + } + return detailsMap; + } + + public String getProviderName() { + return providerName; + } + + public void setUrl(String url) { + this.url = url; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public void setDetails(Map details) { + this.details = details; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute(){ + try{ + ObjectStore result = _storageService.discoverObjectStore(getName(), getUrl(), getProviderName(), getDetails()); + ObjectStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createObjectStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("objectstore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add object storage"); + } + } catch (Exception ex) { + s_logger.error("Exception: ", ex); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java new file mode 100644 index 00000000000..ed305d9689d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmd.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.user.Account; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteObjectStoragePool", description = "Deletes an Object Storage Pool", responseObject = SuccessResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class DeleteObjectStoragePoolCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(DeleteObjectStoragePoolCmd.class.getName()); + + // /////////////////////////////////////////////////// + // ////////////// API parameters ///////////////////// + // /////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, required = true, description = "The Object Storage ID.") + private Long id; + + // /////////////////////////////////////////////////// + // ///////////////// Accessors /////////////////////// + // /////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + // /////////////////////////////////////////////////// + // ///////////// API Implementation/////////////////// + // /////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + boolean result = _storageService.deleteObjectStore(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete object store"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java new file mode 100644 index 00000000000..9d8d8eccc3c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListObjectStoragePoolsCmd.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.log4j.Logger; + +@APICommand(name = "listObjectStoragePools", description = "Lists object storage pools.", responseObject = ObjectStoreResponse.class, since = "4.19.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListObjectStoragePoolsCmd extends BaseListCmd { + public static final Logger s_logger = Logger.getLogger(ListObjectStoragePoolsCmd.class.getName()); + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the object store") + private String storeName; + + @Parameter(name = ApiConstants.PROVIDER, type = CommandType.STRING, description = "the object store provider") + private String provider; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, description = "the ID of the storage pool") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + + public String getStoreName() { + return storeName; + } + + public Long getId() { + return id; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = _queryService.searchForObjectStores(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java new file mode 100644 index 00000000000..497179d25ef --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = UpdateObjectStoragePoolCmd.APINAME, description = "Updates object storage pool", responseObject = ObjectStoreResponse.class, entityType = {ObjectStore.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0") +public class UpdateObjectStoragePoolCmd extends BaseCmd { + public static final String APINAME = "updateObjectStoragePool"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ObjectStoreResponse.class, required = true, description = "Object Store ID") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name for the object store") + private String name; + + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, description = "the url for the object store") + private String url; + + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public String getUrl() { + return url; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ObjectStore result = _storageService.updateObjectStore(getId(), this); + + ObjectStoreResponse storeResponse = null; + if (result != null) { + storeResponse = _responseGenerator.createObjectStoreResponse(result); + storeResponse.setResponseName(getCommandName()); + storeResponse.setObjectName("objectstore"); + setResponseObject(storeResponse); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update object storage"); + } + } + + @Override + public String getCommandName() { + return APINAME; + } + + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java new file mode 100644 index 00000000000..e9a140cf46e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/CreateBucketCmd.java @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceAllocationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCreateCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "createBucket", responseObject = BucketResponse.class, + description = "Creates a bucket in the specified object storage pool. ", responseView = ResponseView.Restricted, + entityType = {Bucket.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class CreateBucketCmd extends BaseAsyncCreateCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(CreateBucketCmd.class.getName()); + private static final String s_name = "createbucketresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ACCOUNT, + type = CommandType.STRING, + description = "the account associated with the bucket. Must be used with the domainId parameter.") + private String accountName; + + @Parameter(name = ApiConstants.PROJECT_ID, + type = CommandType.UUID, + entityType = ProjectResponse.class, + description = "the project associated with the bucket. Mutually exclusive with account parameter") + private Long projectId; + + @Parameter(name = ApiConstants.DOMAIN_ID, + type = CommandType.UUID, + entityType = DomainResponse.class, + description = "the domain ID associated with the bucket. If used with the account parameter" + + " returns the bucket associated with the account for the specified domain.") + private Long domainId; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true,description = "the name of the bucket") + private String bucketName; + + @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, + entityType = ObjectStoreResponse.class, required = true, + description = "Id of the Object Storage Pool where bucket is created") + private long objectStoragePoolId; + + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + private Integer quota; + + @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable bucket encryption") + private boolean encryption; + + @Parameter(name = ApiConstants.VERSIONING, type = CommandType.BOOLEAN, description = "Enable bucket versioning") + private boolean versioning; + + @Parameter(name = ApiConstants.OBJECT_LOCKING, type = CommandType.BOOLEAN, description = "Enable object locking in bucket") + private boolean objectLocking; + + @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING,description = "The Bucket access policy") + private String policy; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public String getBucketName() { + return bucketName; + } + + private Long getProjectId() { + return projectId; + } + + public long getObjectStoragePoolId() { + return objectStoragePoolId; + } + + public Integer getQuota() { + return quota; + } + + public boolean isEncryption() { + return encryption; + } + + public boolean isVersioning() { + return versioning; + } + + public boolean isObjectLocking() { + return objectLocking; + } + + public String getPolicy() { + return policy; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + @Override + public String getCommandName() { + return s_name; + } + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_BUCKET_CREATE; + } + + @Override + public String getEventDescription() { + return "creating bucket: " + getBucketName(); + } + + @Override + public void create() throws ResourceAllocationException { + Bucket bucket = _bucketService.allocBucket(this); + if (bucket != null) { + setEntityId(bucket.getId()); + setEntityUuid(bucket.getUuid()); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create bucket"); + } + } + + @Override + public void execute() { + CallContext.current().setEventDetails("Bucket Id: " + getEntityUuid()); + + Bucket bucket; + try { + bucket = _bucketService.createBucket(this); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); + } + if (bucket != null) { + BucketResponse response = _responseGenerator.createBucketResponse(bucket); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create bucket with name: "+getBucketName()); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java new file mode 100644 index 00000000000..bf9552b779e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "deleteBucket", description = "Deletes an empty Bucket.", responseObject = SuccessResponse.class, entityType = {Bucket.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class DeleteBucketCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(DeleteBucketCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=BucketResponse.class, + required=true, description="The ID of the Bucket") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public long getEntityOwnerId() { + Bucket Bucket = _entityMgr.findById(Bucket.class, getId()); + if (Bucket != null) { + return Bucket.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public Long getApiResourceId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() throws ConcurrentOperationException { + CallContext.current().setEventDetails("Bucket Id: " + this._uuidMgr.getUuid(Bucket.class, getId())); + boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java new file mode 100644 index 00000000000..897b9fc6696 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/ListBucketsCmd.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.log4j.Logger; + +import java.util.List; + +@APICommand(name = "listBuckets", description = "Lists all Buckets.", responseObject = BucketResponse.class, responseView = ResponseView.Restricted, entityType = { + Bucket.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListBucketsCmd extends BaseListTaggedResourcesCmd implements UserCmd { + public static final Logger s_logger = Logger.getLogger(ListBucketsCmd.class.getName()); + + private static final String s_name = "listbucketsresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = BucketResponse.class, description = "the ID of the bucket") + private Long id; + + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, entityType = BucketResponse.class, description = "the IDs of the Buckets, mutually exclusive with id") + private List ids; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "the name of the bucket") + private String bucketName; + + @Parameter(name = ApiConstants.OBJECT_STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "the ID of the object storage pool, available to ROOT admin only", authorized = { + RoleType.Admin}) + private Long objectStorageId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getBucketName() { + return bucketName; + } + + public Long getObjectStorageId() { + return objectStorageId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() { + ListResponse response = _queryService.searchForBuckets(this); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + public List getIds() { + return ids; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java new file mode 100644 index 00000000000..b3b7e00770d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/UpdateBucketCmd.java @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.bucket; + +import com.cloud.exception.ConcurrentOperationException; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.user.Account; +import org.apache.cloudstack.acl.SecurityChecker.AccessType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BucketResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; + +@APICommand(name = "updateBucket", description = "Updates Bucket properties", responseObject = SuccessResponse.class, entityType = {Bucket.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.19.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class UpdateBucketCmd extends BaseCmd { + public static final Logger s_logger = Logger.getLogger(UpdateBucketCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @ACL(accessType = AccessType.OperateEntry) + @Parameter(name=ApiConstants.ID, type=CommandType.UUID, entityType=BucketResponse.class, + required=true, description="The ID of the Bucket") + private Long id; + + @Parameter(name = ApiConstants.VERSIONING, type = CommandType.BOOLEAN, description = "Enable/Disable Bucket Versioning") + private Boolean versioning; + + @Parameter(name = ApiConstants.ENCRYPTION, type = CommandType.BOOLEAN, description = "Enable/Disable Bucket encryption") + private Boolean encryption; + + @Parameter(name = ApiConstants.POLICY, type = CommandType.STRING, description = "Bucket Access Policy") + private String policy; + + @Parameter(name = ApiConstants.QUOTA, type = CommandType.INTEGER,description = "Bucket Quota in GB") + private Integer quota; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Boolean getVersioning() { + return versioning; + } + + public Boolean getEncryption() { + return encryption; + } + + public String getPolicy() { + return policy; + } + + public Integer getQuota() { + return quota; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + public static String getResultObjectName() { + return "bucket"; + } + + @Override + public long getEntityOwnerId() { + Bucket Bucket = _entityMgr.findById(Bucket.class, getId()); + if (Bucket != null) { + return Bucket.getAccountId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public Long getApiResourceId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Bucket; + } + + @Override + public void execute() throws ConcurrentOperationException { + CallContext.current().setEventDetails("Bucket Id: " + this._uuidMgr.getUuid(Bucket.class, getId())); + boolean result = false; + try { + result = _bucketService.updateBucket(this, CallContext.current().getCallingAccount()); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Error while updating bucket. "+e.getMessage()); + } + if(result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update bucket"); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java new file mode 100644 index 00000000000..b75f3604324 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/BucketResponse.java @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponseWithTagInformation; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.storage.object.Bucket; + +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.Set; + +@EntityReference(value = Bucket.class) +@SuppressWarnings("unused") +public class BucketResponse extends BaseResponseWithTagInformation implements ControlledViewEntityResponse, ControlledEntityResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the Bucket") + private String id; + @SerializedName(ApiConstants.NAME) + @Param(description = "name of the Bucket") + private String name; + @SerializedName(ApiConstants.CREATED) + @Param(description = "the date the Bucket was created") + private Date created; + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "the account associated with the Bucket") + private String accountName; + @SerializedName(ApiConstants.PROJECT_ID) + @Param(description = "the project id of the bucket") + private String projectId; + @SerializedName(ApiConstants.PROJECT) + @Param(description = "the project name of the bucket") + private String projectName; + @SerializedName(ApiConstants.DOMAIN_ID) + @Param(description = "the ID of the domain associated with the bucket") + private String domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "the domain associated with the bucket") + private String domainName; + @SerializedName(ApiConstants.OBJECT_STORAGE_ID) + @Param(description = "id of the object storage hosting the Bucket; returned to admin user only") + private String objectStoragePoolId; + + @SerializedName(ApiConstants.OBJECT_STORAGE) + @Param(description = "Name of the object storage hosting the Bucket; returned to admin user only") + private String objectStoragePool; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "Total size of objects in Bucket") + private Long size; + + @SerializedName(ApiConstants.STATE) + @Param(description = "State of the Bucket") + private String state; + + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Bucket Quota in GB") + private Integer quota; + + @SerializedName(ApiConstants.ENCRYPTION) + @Param(description = "Bucket Encryption") + private Boolean encryption; + + @SerializedName(ApiConstants.VERSIONING) + @Param(description = "Bucket Versioning") + private Boolean versioning; + + @SerializedName(ApiConstants.OBJECT_LOCKING) + @Param(description = "Bucket Object Locking") + private Boolean objectLock; + + @SerializedName(ApiConstants.POLICY) + @Param(description = "Bucket Access Policy") + private String policy; + + @SerializedName(ApiConstants.URL) + @Param(description = "Bucket URL") + private String bucketURL; + + @SerializedName(ApiConstants.ACCESS_KEY) + @Param(description = "Bucket Access Key") + private String accessKey; + + @SerializedName(ApiConstants.SECRET_KEY) + @Param(description = "Bucket Secret Key") + private String secretKey; + + @SerializedName(ApiConstants.PROVIDER) + @Param(description = "Object storage provider") + private String provider; + + public BucketResponse() { + tags = new LinkedHashSet(); + } + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public void setName(String name) { + this.name = name; + } + + public void setCreated(Date created) { + this.created = created; + } + + @Override + public void setAccountName(String accountName) { + this.accountName = accountName; + } + + @Override + public void setDomainId(String domainId) { + this.domainId = domainId; + } + + @Override + public void setDomainName(String domainName) { + this.domainName = domainName; + } + + @Override + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + @Override + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setTags(Set tags) { + this.tags = tags; + } + + public void setObjectStoragePoolId(String objectStoragePoolId) { + this.objectStoragePoolId = objectStoragePoolId; + } + + public String getName() { + return name; + } + + public Date getCreated() { + return created; + } + + public String getAccountName() { + return accountName; + } + + public String getProjectId() { + return projectId; + } + + public String getProjectName() { + return projectName; + } + + public String getDomainId() { + return domainId; + } + + public String getDomainName() { + return domainName; + } + + public String getObjectStoragePoolId() { + return objectStoragePoolId; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + public long getQuota() { + return quota; + } + + public void setQuota(Integer quota) { + this.quota = quota; + } + + public boolean isVersioning() { + return versioning; + } + + public void setVersioning(boolean versioning) { + this.versioning = versioning; + } + + public boolean isEncryption() { + return encryption; + } + + public void setEncryption(boolean encryption) { + this.encryption = encryption; + } + + public boolean isObjectLock() { + return objectLock; + } + + public void setObjectLock(boolean objectLock) { + this.objectLock = objectLock; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(String policy) { + this.policy = policy; + } + + public String getBucketURL() { + return bucketURL; + } + + public void setBucketURL(String bucketURL) { + this.bucketURL = bucketURL; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setState(Bucket.State state) { + this.state = state.toString(); + } + + public String getState() { + return state; + } + + public void setObjectStoragePool(String objectStoragePool) { + this.objectStoragePool = objectStoragePool; + } + + public String getObjectStoragePool() { + return objectStoragePool; + } + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java new file mode 100644 index 00000000000..e4030799aa7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/ObjectStoreResponse.java @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import org.apache.cloudstack.storage.object.ObjectStore; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.BaseResponseWithAnnotations; +import org.apache.cloudstack.api.EntityReference; + +@EntityReference(value = ObjectStore.class) +public class ObjectStoreResponse extends BaseResponseWithAnnotations { + @SerializedName("id") + @Param(description = "the ID of the object store") + private String id; + + @SerializedName("name") + @Param(description = "the name of the object store") + private String name; + + @SerializedName("url") + @Param(description = "the url of the object store") + private String url; + + @SerializedName("providername") + @Param(description = "the provider name of the object store") + private String providerName; + + @SerializedName("storagetotal") + @Param(description = "the total size of the object store") + private Long storageTotal; + + @SerializedName("storageused") + @Param(description = "the object store currently used size") + private Long storageUsed; + + public ObjectStoreResponse() { + } + + @Override + public String getObjectId() { + return this.getId(); + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getProviderName() { + return providerName; + } + + public void setProviderName(String providerName) { + this.providerName = providerName; + } + + public Long getStorageTotal() { + return storageTotal; + } + + public void setStorageTotal(Long storageTotal) { + this.storageTotal = storageTotal; + } + + public Long getStorageUsed() { + return storageUsed; + } + + public void setStorageUsed(Long storageUsed) { + this.storageUsed = storageUsed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 6f404452598..17dd14da84c 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -28,6 +28,7 @@ import org.apache.cloudstack.api.command.admin.resource.icon.ListResourceIconCmd import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResultsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; @@ -36,6 +37,7 @@ import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; import org.apache.cloudstack.api.command.user.address.ListQuarantinedIpsCmd; import org.apache.cloudstack.api.command.user.affinitygroup.ListAffinityGroupsCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; import org.apache.cloudstack.api.command.user.job.ListAsyncJobsCmd; @@ -56,6 +58,7 @@ import org.apache.cloudstack.api.command.user.volume.ListVolumesCmd; import org.apache.cloudstack.api.command.user.zone.ListZonesCmd; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; @@ -68,6 +71,7 @@ import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -190,4 +194,8 @@ public interface QueryService { ListResponse listSnapshots(ListSnapshotsCmd cmd); SnapshotResponse listSnapshot(CopySnapshotCmd cmd); + + ListResponse searchForObjectStores(ListObjectStoragePoolsCmd listObjectStoragePoolsCmd); + + ListResponse searchForBuckets(ListBucketsCmd listBucketsCmd); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java b/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java new file mode 100644 index 00000000000..c821dbac589 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/Bucket.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.acl.ControlledEntity; +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Bucket extends ControlledEntity, Identity, InternalIdentity { + + long getObjectStoreId(); + + Date getCreated(); + + State getState(); + + void setName(String name); + + Long getSize(); + + Integer getQuota(); + + boolean isVersioning(); + + boolean isEncryption(); + + boolean isObjectLock(); + + String getPolicy(); + + String getBucketURL(); + + String getAccessKey(); + + String getSecretKey(); + + public enum State { + Allocated, Created, Destroyed; + @Override + public String toString() { + return this.name(); + } + + public boolean equals(String status) { + return this.toString().equalsIgnoreCase(status); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java new file mode 100644 index 00000000000..7e1361d1e71 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.cloud.exception.ResourceAllocationException; +import com.cloud.user.Account; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; + +public interface BucketApiService { + + + /** + * Creates the database object for a Bucket based on the given criteria + * + * @param cmd + * the API command wrapping the criteria (account/domainId [admin only], zone, diskOffering, snapshot, + * name) + * @return the Bucket object + */ + Bucket allocBucket(CreateBucketCmd cmd) throws ResourceAllocationException; + + /** + * Creates the Bucket based on the given criteria + * + * @param cmd + * the API command wrapping the criteria (account/domainId [admin only], zone, diskOffering, snapshot, + * name) + * @return the Bucket object + */ + Bucket createBucket(CreateBucketCmd cmd); + + boolean deleteBucket(long bucketId, Account caller); + + boolean updateBucket(UpdateBucketCmd cmd, Account caller); + + void getBucketUsage(); +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java b/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java new file mode 100644 index 00000000000..47741fc67f4 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStore.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +public interface ObjectStore extends Identity, InternalIdentity { + + /** + * @return name of the object store. + */ + String getName(); + + /** + * @return object store provider name + */ + String getProviderName(); + + /** + * + * @return uri + */ + String getUrl(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java index 48cff3076fd..5e0f03ff583 100644 --- a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java +++ b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java @@ -45,6 +45,7 @@ public class UsageTypes { public static final int VOLUME_SECONDARY = 26; public static final int VM_SNAPSHOT_ON_PRIMARY = 27; public static final int BACKUP = 28; + public static final int BUCKET = 29; public static List listUsageTypes() { List responseList = new ArrayList(); @@ -70,6 +71,7 @@ public class UsageTypes { responseList.add(new UsageTypeResponse(VOLUME_SECONDARY, "Volume on secondary storage usage")); responseList.add(new UsageTypeResponse(VM_SNAPSHOT_ON_PRIMARY, "VM Snapshot on primary storage usage")); responseList.add(new UsageTypeResponse(BACKUP, "Backup storage usage")); + responseList.add(new UsageTypeResponse(BUCKET, "Bucket storage usage")); return responseList; } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..f64df167e25 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/AddObjectStoragePoolCmdTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.exception.DiscoveryException; +import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.anyObject; + +@RunWith(MockitoJUnitRunner.class) +public class AddObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(AddObjectStoragePoolCmdTest.class.getName()); + + @Mock + StorageService storageService; + + @Mock + ObjectStore objectStore; + + @Mock + ResponseGenerator responseGenerator; + + @Spy + AddObjectStoragePoolCmd addObjectStoragePoolCmdSpy; + + String name = "testObjStore"; + + String url = "testURL"; + + String provider = "Simulator"; + + Map details; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + details = new HashMap<>(); + addObjectStoragePoolCmdSpy = Mockito.spy(new AddObjectStoragePoolCmd()); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "name", name); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "url", url); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "providerName", provider); + ReflectionTestUtils.setField(addObjectStoragePoolCmdSpy, "details", details); + addObjectStoragePoolCmdSpy._storageService = storageService; + addObjectStoragePoolCmdSpy._responseGenerator = responseGenerator; + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testAddObjectStore() throws DiscoveryException { + Mockito.doReturn(objectStore).when(storageService).discoverObjectStore(Mockito.anyString(), + Mockito.anyString(), Mockito.anyString(), anyObject()); + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(anyObject()); + addObjectStoragePoolCmdSpy.execute(); + + Mockito.verify(storageService, Mockito.times(1)) + .discoverObjectStore(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..35be56d0c75 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DeleteObjectStoragePoolCmdTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.StorageService; +import org.apache.cloudstack.context.CallContext; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +public class DeleteObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(DeleteObjectStoragePoolCmdTest.class.getName()); + @Mock + private StorageService storageService; + + @Spy + DeleteObjectStoragePoolCmd deleteObjectStoragePoolCmd; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + deleteObjectStoragePoolCmd = Mockito.spy(new DeleteObjectStoragePoolCmd()); + deleteObjectStoragePoolCmd._storageService = storageService; + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testDeleteObjectStore() { + Mockito.doReturn(true).when(storageService).deleteObjectStore(deleteObjectStoragePoolCmd); + deleteObjectStoragePoolCmd.execute(); + Mockito.verify(storageService, Mockito.times(1)) + .deleteObjectStore(deleteObjectStoragePoolCmd); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java new file mode 100644 index 00000000000..ef66c2a1a64 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/UpdateObjectStoragePoolCmdTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.storage.StorageService; +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.mockito.ArgumentMatchers.anyObject; + +public class UpdateObjectStoragePoolCmdTest { + public static final Logger s_logger = Logger.getLogger(UpdateObjectStoragePoolCmdTest.class.getName()); + + @Mock + private StorageService storageService; + + @Spy + UpdateObjectStoragePoolCmd updateObjectStoragePoolCmd; + + @Mock + ObjectStore objectStore; + + @Mock + ResponseGenerator responseGenerator; + + private String name = "testObjStore"; + + private String url = "testURL"; + + private String provider = "Simulator"; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + updateObjectStoragePoolCmd = Mockito.spy(new UpdateObjectStoragePoolCmd()); + updateObjectStoragePoolCmd._storageService = storageService; + updateObjectStoragePoolCmd._responseGenerator = responseGenerator; + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "name", name); + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "url", url); + ReflectionTestUtils.setField(updateObjectStoragePoolCmd, "id", 1L); + } + + @After + public void tearDown() throws Exception { + CallContext.unregister(); + } + + @Test + public void testUpdateObjectStore() { + Mockito.doReturn(objectStore).when(storageService).updateObjectStore(1L, updateObjectStoragePoolCmd); + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + Mockito.doReturn(objectStoreResponse).when(responseGenerator).createObjectStoreResponse(anyObject()); + updateObjectStoragePoolCmd.execute(); + Mockito.verify(storageService, Mockito.times(1)) + .updateObjectStore(1L, updateObjectStoragePoolCmd); + } + +} diff --git a/client/pom.xml b/client/pom.xml index de9e910b978..0451e8e09e8 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -577,6 +577,21 @@ cloud-plugin-shutdown ${project.version} + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-object-minio + ${project.version} + + + org.apache.cloudstack + cloud-plugin-storage-object-simulator + ${project.version} + diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java new file mode 100644 index 00000000000..366c7c8fb41 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/BucketInfo.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +import org.apache.cloudstack.storage.object.Bucket; + +public interface BucketInfo extends DataObject, Bucket { + + void addPayload(Object data); + + Object getPayload(); + + Bucket getBucket(); +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java index 3e5761ef37f..41c1d940745 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProvider.java @@ -31,7 +31,7 @@ public interface DataStoreProvider { String DEFAULT_PRIMARY = "DefaultPrimary"; enum DataStoreProviderType { - PRIMARY, IMAGE, ImageCache + PRIMARY, IMAGE, ImageCache, OBJECT } DataStoreLifeCycle getDataStoreLifeCycle(); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java index e476d8f3b35..379911a54fb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreProviderManager.java @@ -33,4 +33,6 @@ public interface DataStoreProviderManager extends Manager, DataStoreProviderApiS DataStoreProvider getDefaultCacheDataStoreProvider(); List getProviders(); + + DataStoreProvider getDefaultObjectStoreProvider(); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java new file mode 100644 index 00000000000..691da7ecc8d --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStorageService.java @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface ObjectStorageService { + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java new file mode 100644 index 00000000000..71b3762b441 --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectStoreProvider.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.engine.subsystem.api.storage; + +public interface ObjectStoreProvider extends DataStoreProvider { + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java new file mode 100644 index 00000000000..9ee94b083cf --- /dev/null +++ b/engine/api/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreEntity.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; + +import java.util.List; +import java.util.Map; + +public interface ObjectStoreEntity extends DataStore, ObjectStore { + Bucket createBucket(Bucket bucket, boolean objectLock); + + List listBuckets(); + + boolean createUser(long accountId); + + boolean deleteBucket(String name); + + boolean setBucketEncryption(String name); + + boolean deleteBucketEncryption(String name); + + boolean setBucketVersioning(String name); + + boolean deleteBucketVersioning(String name); + + void setBucketPolicy(String name, String policy); + + void setQuota(String name, int quota); + + Map getAllBucketsUsage(); +} diff --git a/engine/pom.xml b/engine/pom.xml index b2f41834403..6ecfb4bebf4 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -55,6 +55,7 @@ storage/configdrive storage/datamotion storage/image + storage/object storage/snapshot storage/volume userdata/cloud-init diff --git a/engine/schema/src/main/java/com/cloud/storage/BucketVO.java b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java new file mode 100644 index 00000000000..181b02e5a1b --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/BucketVO.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage; + +import com.cloud.utils.db.GenericDao; +import com.google.gson.annotations.Expose; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "bucket") +public class BucketVO implements Bucket { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "account_id") + long accountId; + + @Column(name = "domain_id") + long domainId; + + @Column(name = "object_store_id") + long objectStoreId; + + @Expose + @Column(name = "name") + String name; + + @Expose + @Column(name = "state", updatable = true, nullable = false) + @Enumerated(value = EnumType.STRING) + private State state; + + @Column(name = "size") + Long size; + + @Column(name = "quota") + Integer quota; + + @Column(name = "versioning") + boolean versioning; + + @Column(name = "encryption") + boolean encryption; + + @Column(name = "object_lock") + boolean objectLock; + + @Column(name = "policy") + String policy; + + @Column(name = "bucket_url") + String bucketURL; + + @Column(name = "access_key") + String accessKey; + + @Column(name = "secret_key") + String secretKey; + + @Column(name = GenericDao.CREATED_COLUMN) + Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + Date removed; + + @Column(name = "uuid") + String uuid; + + public BucketVO() { + } + + public BucketVO(long accountId, long domainId, long objectStoreId, String name, Integer quota, boolean versioning, + boolean encryption, boolean objectLock, String policy) + { + this.accountId = accountId; + this.domainId = domainId; + this.objectStoreId = objectStoreId; + this.name = name; + state = State.Allocated; + uuid = UUID.randomUUID().toString(); + this.quota = quota; + this.versioning = versioning; + this.encryption = encryption; + this.objectLock = objectLock; + this.policy = policy; + this.size = 0L; + } + + @Override + public long getId() { + return id; + } + + @Override + public long getAccountId() { + return accountId; + } + + @Override + public long getDomainId() { + return domainId; + } + + @Override + public long getObjectStoreId() { + return objectStoreId; + } + + @Override + public String getName() { + return name; + } + + public Long getSize() { + return size; + } + + @Override + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + @Override + public State getState() { + return state; + } + + @Override + public void setName(String name) { + this.name = name; + } + + public void setState(State state) { + this.state = state; + } + + @Override + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public Integer getQuota() { + return quota; + } + + public void setQuota(Integer quota) { + this.quota = quota; + } + + public boolean isVersioning() { + return versioning; + } + + public void setVersioning(boolean versioning) { + this.versioning = versioning; + } + + public boolean isEncryption() { + return encryption; + } + + public void setEncryption(boolean encryption) { + this.encryption = encryption; + } + + public boolean isObjectLock() { + return objectLock; + } + + public void setObjectLock(boolean objectLock) { + this.objectLock = objectLock; + } + + public String getPolicy() { + return policy; + } + + public void setPolicy(String policy) { + this.policy = policy; + } + + public String getBucketURL() { + return bucketURL; + } + public void setBucketURL(String bucketURL) { + this.bucketURL = bucketURL; + } + + public String getAccessKey() { + return accessKey; + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey; + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public void setSize(Long size) { + this.size = size; + } + + @Override + public Class getEntityType() { + return Bucket.class; + } + + @Override + public String toString() { + return String.format("Bucket %s", new ToStringBuilder(this, ToStringStyle.JSON_STYLE).append("uuid", getUuid()).append("name", getName()) + .append("ObjectStoreId", getObjectStoreId()).toString()); + } +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java new file mode 100644 index 00000000000..f45f28b5c2c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.dao; + +import com.cloud.storage.BucketVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface BucketDao extends GenericDao { + List listByObjectStoreId(long objectStoreId); + + List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId); + + List searchByIds(Long[] ids); +} diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java new file mode 100644 index 00000000000..83b5f6bdb74 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/storage/dao/BucketDaoImpl.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.storage.dao; + +import com.cloud.storage.BucketVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.naming.ConfigurationException; +import java.util.List; +import java.util.Map; + +@Component +public class BucketDaoImpl extends GenericDaoBase implements BucketDao { + public static final Logger s_logger = Logger.getLogger(BucketDaoImpl.class.getName()); + private SearchBuilder searchFilteringStoreId; + + private SearchBuilder bucketSearch; + + private static final String STORE_ID = "store_id"; + private static final String STATE = "state"; + private static final String ACCOUNT_ID = "account_id"; + + protected BucketDaoImpl() { + + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + searchFilteringStoreId = createSearchBuilder(); + searchFilteringStoreId.and(STORE_ID, searchFilteringStoreId.entity().getObjectStoreId(), SearchCriteria.Op.EQ); + searchFilteringStoreId.and(ACCOUNT_ID, searchFilteringStoreId.entity().getAccountId(), SearchCriteria.Op.EQ); + searchFilteringStoreId.and(STATE, searchFilteringStoreId.entity().getState(), SearchCriteria.Op.NEQ); + searchFilteringStoreId.done(); + + bucketSearch = createSearchBuilder(); + bucketSearch.and("idIN", bucketSearch.entity().getId(), SearchCriteria.Op.IN); + bucketSearch.done(); + + return true; + } + @Override + public List listByObjectStoreId(long objectStoreId) { + SearchCriteria sc = searchFilteringStoreId.create(); + sc.setParameters(STORE_ID, objectStoreId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return listBy(sc); + } + + @Override + public List listByObjectStoreIdAndAccountId(long objectStoreId, long accountId) { + SearchCriteria sc = searchFilteringStoreId.create(); + sc.setParameters(STORE_ID, objectStoreId); + sc.setParameters(ACCOUNT_ID, accountId); + sc.setParameters(STATE, BucketVO.State.Destroyed); + return listBy(sc); + } + + @Override + public List searchByIds(Long[] ids) { + SearchCriteria sc = bucketSearch.create(); + sc.setParameters("idIN", ids); + return search(sc, null, null, false); + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java b/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java new file mode 100644 index 00000000000..ab5fcfc493c --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/BucketStatisticsVO.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage; + +import org.apache.cloudstack.api.InternalIdentity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "bucket_statistics") +public class BucketStatisticsVO implements InternalIdentity { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private Long id; + + @Column(name = "account_id", updatable = false) + private long accountId; + + @Column(name = "bucket_id", updatable = false) + private long bucketId; + + @Column(name = "size") + private long size; + + protected BucketStatisticsVO() { + } + + public BucketStatisticsVO(long accountId, long bucketId) { + this.accountId = accountId; + this.bucketId = bucketId; + } + + public long getAccountId() { + return accountId; + } + + public long getBucketId() { + return bucketId; + } + + @Override + public long getId() { + return id; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java new file mode 100644 index 00000000000..48388af9446 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDao.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.utils.db.GenericDao; + +import java.util.List; + +public interface BucketStatisticsDao extends GenericDao { + BucketStatisticsVO findBy(long accountId, long bucketId); + + BucketStatisticsVO lock(long accountId, long bucketId); + + List listBy(long accountId); +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java new file mode 100644 index 00000000000..2261389eab6 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/BucketStatisticsDaoImpl.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import java.util.List; + +@Component +public class BucketStatisticsDaoImpl extends GenericDaoBase implements BucketStatisticsDao { + private static final Logger s_logger = Logger.getLogger(BucketStatisticsDaoImpl.class); + private final SearchBuilder AllFieldsSearch; + private final SearchBuilder AccountSearch; + + public BucketStatisticsDaoImpl() { + AccountSearch = createSearchBuilder(); + AccountSearch.and("account", AccountSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AccountSearch.done(); + + AllFieldsSearch = createSearchBuilder(); + AllFieldsSearch.and("account", AllFieldsSearch.entity().getAccountId(), SearchCriteria.Op.EQ); + AllFieldsSearch.and("bucket", AllFieldsSearch.entity().getBucketId(), SearchCriteria.Op.EQ); + AllFieldsSearch.done(); + } + + @Override + public BucketStatisticsVO findBy(long accountId, long bucketId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("account", accountId); + sc.setParameters("bucket", bucketId); + return findOneBy(sc); + } + + @Override + public BucketStatisticsVO lock(long accountId, long bucketId) { + SearchCriteria sc = AllFieldsSearch.create(); + sc.setParameters("account", accountId); + sc.setParameters("bucket", bucketId); + return lockOneRandomRow(sc, true); + } + + @Override + public List listBy(long accountId) { + SearchCriteria sc = AccountSearch.create(); + sc.setParameters("account", accountId); + return search(sc, null); + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java index 4099b3ada0d..ea490e60f9e 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDao.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.dao; +import com.cloud.usage.BucketStatisticsVO; import com.cloud.usage.UsageVO; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -45,6 +46,12 @@ public interface UsageDao extends GenericDao { Long getLastUserStatsId(); + Long getLastBucketStatsId(); + + void saveBucketStats(List userStats); + + void updateBucketStats(List userStats); + List listPublicTemplatesByAccount(long accountId); Long getLastVmDiskStatsId(); diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java index 4553ed822b4..0d9e727abe2 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageDaoImpl.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.dao; +import com.cloud.usage.BucketStatisticsVO; import com.cloud.usage.UsageVO; import com.cloud.user.AccountVO; import com.cloud.user.UserStatisticsVO; @@ -82,6 +83,13 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage + "WHERE cloud_usage.usage_type = ? AND cloud_usage.account_id = ? AND cloud_usage.start_date >= ? AND cloud_usage.end_date <= ? " + "GROUP BY cloud_usage.usage_id "; + private static final String GET_LAST_BUCKET_STATS_ID = "SELECT id FROM cloud_usage.bucket_statistics ORDER BY id DESC LIMIT 1"; + + private static final String INSERT_BUCKET_STATS = "INSERT INTO cloud_usage.bucket_statistics (id, account_id, bucket_id, size) VALUES (?,?,?,?)"; + + private static final String UPDATE_BUCKET_STATS = "UPDATE cloud_usage.bucket_statistics SET size=? WHERE id=?"; + + public UsageDaoImpl() { } @@ -285,6 +293,69 @@ public class UsageDaoImpl extends GenericDaoBase implements Usage return null; } + @Override + public Long getLastBucketStatsId() { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt = null; + String sql = GET_LAST_BUCKET_STATS_ID; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return Long.valueOf(rs.getLong(1)); + } + } catch (Exception ex) { + s_logger.error("error getting last bucket stats id", ex); + } + return null; + } + + @Override + public void saveBucketStats(List bucketStats) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + txn.start(); + String sql = INSERT_BUCKET_STATS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (BucketStatisticsVO bucketStat : bucketStats) { + pstmt.setLong(1, bucketStat.getId()); + pstmt.setLong(2, bucketStat.getAccountId()); + pstmt.setLong(3, bucketStat.getBucketId()); + pstmt.setLong(4, bucketStat.getSize()); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error saving bucket stats to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } + + @Override + public void updateBucketStats(List bucketStats) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + try { + txn.start(); + String sql = UPDATE_BUCKET_STATS; + PreparedStatement pstmt = null; + pstmt = txn.prepareAutoCloseStatement(sql); // in reality I just want CLOUD_USAGE dataSource connection + for (BucketStatisticsVO bucketStat : bucketStats) { + pstmt.setLong(1, bucketStat.getSize()); + pstmt.setLong(2, bucketStat.getId()); + pstmt.addBatch(); + } + pstmt.executeBatch(); + txn.commit(); + } catch (Exception ex) { + txn.rollback(); + s_logger.error("error updating bucket stats to cloud_usage db", ex); + throw new CloudRuntimeException(ex.getMessage()); + } + } + @Override public List listPublicTemplatesByAccount(long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java new file mode 100644 index 00000000000..94f6b5ec372 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDao.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.api.response.ObjectStoreResponse; + +import java.util.List; + +public interface ObjectStoreDao extends GenericDao { + ObjectStoreVO findByName(String name); + + List findByProvider(String provider); + + ObjectStoreVO findByUrl(String url); + + List listObjectStores(); + + List searchByIds(Long[] osIds); + + ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store); + + ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store); + + Integer countAllObjectStores(); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java new file mode 100644 index 00000000000..51abde013b6 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDaoImpl.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreDaoImpl extends GenericDaoBase implements ObjectStoreDao { + private SearchBuilder nameSearch; + private SearchBuilder providerSearch; + @Inject + private ConfigurationDao _configDao; + private final SearchBuilder osSearch; + + private SearchBuilder urlSearch; + + protected ObjectStoreDaoImpl() { + osSearch = createSearchBuilder(); + osSearch.and("idIN", osSearch.entity().getId(), SearchCriteria.Op.IN); + osSearch.done(); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + nameSearch = createSearchBuilder(); + nameSearch.and("name", nameSearch.entity().getName(), SearchCriteria.Op.EQ); + nameSearch.done(); + + providerSearch = createSearchBuilder(); + providerSearch.and("providerName", providerSearch.entity().getProviderName(), SearchCriteria.Op.EQ); + providerSearch.done(); + + urlSearch = createSearchBuilder(); + urlSearch.and("url", urlSearch.entity().getUrl(), SearchCriteria.Op.EQ); + urlSearch.done(); + + return true; + } + + @Override + public ObjectStoreVO findByName(String name) { + SearchCriteria sc = nameSearch.create(); + sc.setParameters("name", name); + return findOneBy(sc); + } + + @Override + public List findByProvider(String provider) { + SearchCriteria sc = providerSearch.create(); + sc.setParameters("providerName", provider); + return listBy(sc); + } + + @Override + public ObjectStoreVO findByUrl(String url) { + SearchCriteria sc = urlSearch.create(); + sc.setParameters("url", url); + return findOneBy(sc); + } + + @Override + public List listObjectStores() { + SearchCriteria sc = createSearchCriteria(); + return listBy(sc); + } + + @Override + public List searchByIds(Long[] osIds) { + // set detail batch query size + int DETAILS_BATCH_SIZE = 2000; + String batchCfg = _configDao.getValue("detail.batch.query.size"); + if (batchCfg != null) { + DETAILS_BATCH_SIZE = Integer.parseInt(batchCfg); + } + // query details by batches + List osList = new ArrayList<>(); + // query details by batches + int curr_index = 0; + if (osIds.length > DETAILS_BATCH_SIZE) { + while ((curr_index + DETAILS_BATCH_SIZE) <= osIds.length) { + Long[] ids = new Long[DETAILS_BATCH_SIZE]; + for (int k = 0, j = curr_index; j < curr_index + DETAILS_BATCH_SIZE; j++, k++) { + ids[k] = osIds[j]; + } + SearchCriteria sc = osSearch.create(); + sc.setParameters("idIN", ids); + List stores = searchIncludingRemoved(sc, null, null, false); + if (stores != null) { + osList.addAll(stores); + } + curr_index += DETAILS_BATCH_SIZE; + } + } + if (curr_index < osIds.length) { + int batch_size = (osIds.length - curr_index); + // set the ids value + Long[] ids = new Long[batch_size]; + for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) { + ids[k] = osIds[j]; + } + SearchCriteria sc = osSearch.create(); + sc.setParameters("idIN", ids); + List stores = searchIncludingRemoved(sc, null, null, false); + if (stores != null) { + osList.addAll(stores); + } + } + return osList; + } + + @Override + public ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store) { + ObjectStoreResponse osResponse = new ObjectStoreResponse(); + osResponse.setId(store.getUuid()); + osResponse.setName(store.getName()); + osResponse.setProviderName(store.getProviderName()); + String url = store.getUrl(); + osResponse.setUrl(url); + osResponse.setObjectName("objectstore"); + return osResponse; + } + + @Override + public ObjectStoreResponse setObjectStoreResponse(ObjectStoreResponse storeData, ObjectStoreVO store) { + return storeData; + } + + @Override + public Integer countAllObjectStores() { + SearchCriteria sc = createSearchCriteria(); + return getCount(sc); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java new file mode 100644 index 00000000000..1f4047f8f90 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailVO.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import org.apache.cloudstack.api.ResourceDetail; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +@Entity +@Table(name = "object_store_details") +public class ObjectStoreDetailVO implements ResourceDetail { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + long id; + + @Column(name = "store_id") + long resourceId; + + @Column(name = "name") + String name; + + @Column(name = "value") + String value; + + public ObjectStoreDetailVO() { + } + public ObjectStoreDetailVO(long storeId, String name, String value) { + this.resourceId = storeId; + this.name = name; + this.value = value; + } + @Override + public long getId() { + return id; + } + + @Override + public long getResourceId() { + return resourceId; + } + + @Override + public String getName() { + return name; + } + + @Override + public String getValue() { + return value; + } + + @Override + public boolean isDisplay() { + return true; + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java new file mode 100644 index 00000000000..170a28af502 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDao; + +import java.util.Map; + +public interface ObjectStoreDetailsDao extends GenericDao, ResourceDetailsDao { + + void update(long storeId, Map details); + + Map getDetails(long storeId); + + void deleteDetails(long storeId); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java new file mode 100644 index 00000000000..e1000e5d045 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreDetailsDaoImpl.java @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.crypt.DBEncryptionUtil; +import com.cloud.utils.db.QueryBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.resourcedetail.ResourceDetailsDaoBase; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreDetailsDaoImpl extends ResourceDetailsDaoBase implements ObjectStoreDetailsDao { + + protected final SearchBuilder storeSearch; + + public ObjectStoreDetailsDaoImpl() { + super(); + storeSearch = createSearchBuilder(); + storeSearch.and("store", storeSearch.entity().getResourceId(), Op.EQ); + storeSearch.done(); + } + + @Override + public void update(long storeId, Map details) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + txn.start(); + expunge(sc); + for (Map.Entry entry : details.entrySet()) { + ObjectStoreDetailVO detail = new ObjectStoreDetailVO(storeId, entry.getKey(), entry.getValue()); + persist(detail); + } + txn.commit(); + } + + @Override + public Map getDetails(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + List details = listBy(sc); + Map detailsMap = new HashMap(); + for (ObjectStoreDetailVO detail : details) { + String name = detail.getName(); + String value = detail.getValue(); + if (name.equals(ApiConstants.KEY)) { + value = DBEncryptionUtil.decrypt(value); + } + detailsMap.put(name, value); + } + + return detailsMap; + } + + @Override + public void deleteDetails(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store", storeId); + + List results = search(sc, null); + for (ObjectStoreDetailVO result : results) { + remove(result.getId()); + } + } + + @Override + public ObjectStoreDetailVO findDetail(long storeId, String name) { + QueryBuilder sc = QueryBuilder.create(ObjectStoreDetailVO.class); + sc.and(sc.entity().getResourceId(), Op.EQ, storeId); + sc.and(sc.entity().getName(), Op.EQ, name); + return sc.find(); + } + + @Override + public void addDetail(long resourceId, String key, String value, boolean display) { + // ToDo: Add Display + super.addDetail(new ObjectStoreDetailVO(resourceId, key, value)); + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java new file mode 100644 index 00000000000..885cbfd98ab --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ObjectStoreVO.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import org.apache.cloudstack.storage.object.ObjectStore; +import com.cloud.utils.db.GenericDao; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.TableGenerator; +import javax.persistence.Transient; +import java.util.Date; +import java.util.Map; + +@Entity +@Table(name = "object_store") +public class ObjectStoreVO implements ObjectStore { + @Id + @TableGenerator(name = "object_store_sq", table = "sequence", pkColumnName = "name", valueColumnName = "value", pkColumnValue = "object_store_seq", allocationSize = 1) + @Column(name = "id", nullable = false) + private long id; + + @Column(name = "name", nullable = false) + private String name; + + @Column(name = "uuid", nullable = false) + private String uuid; + + @Column(name = "url", nullable = false, length = 2048) + private String url; + + @Column(name = "object_provider_name", nullable = false) + private String providerName; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + @Column(name = "total_size") + private Long totalSize; + + @Column(name = "used_bytes") + private Long usedBytes; + + @Transient + Map details; + + @Override + public long getId() { + return this.id; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public String getProviderName() { + return this.providerName; + } + + public void setName(String name) { + this.name = name; + } + + public void setProviderName(String provider) { + this.providerName = provider; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + @Override + public String getUuid() { + return this.uuid; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public Long getTotalSize() { + return totalSize; + } + + public void setTotalSize(Long totalSize) { + this.totalSize = totalSize; + } + + public Long getUsedBytes() { + return usedBytes; + } + + public void setUsedBytes(Long usedBytes) { + this.usedBytes = usedBytes; + } + + public void setDetails(Map details) { + this.details = details; + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index f0c5e7630e1..97da7e83e7b 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -283,4 +283,6 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index ec6972ccc59..b36814528c0 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -233,3 +233,69 @@ ALTER TABLE `cloud`.`storage_pool_tags` MODIFY tag text NOT NULL; ALTER TABLE `cloud`.`host_tags` ADD COLUMN is_tag_a_rule int(1) UNSIGNED not null DEFAULT 0; ALTER TABLE `cloud`.`host_tags` MODIFY tag text NOT NULL; + +DROP TABLE IF EXISTS `cloud`.`object_store`; +CREATE TABLE `cloud`.`object_store` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `name` varchar(255) NOT NULL COMMENT 'name of object store', + `object_provider_name` varchar(255) NOT NULL COMMENT 'id of object_store_provider', + `url` varchar(255) NOT NULL COMMENT 'url of the object store', + `uuid` varchar(255) COMMENT 'uuid of object store', + `created` datetime COMMENT 'date the object store first signed on', + `removed` datetime COMMENT 'date removed if not null', + `total_size` bigint unsigned COMMENT 'storage total size statistics', + `used_bytes` bigint unsigned COMMENT 'storage available bytes statistics', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`object_store_details`; +CREATE TABLE `cloud`.`object_store_details` ( + `id` bigint unsigned UNIQUE NOT NULL AUTO_INCREMENT COMMENT 'id', + `store_id` bigint unsigned NOT NULL COMMENT 'store the detail is related to', + `name` varchar(255) NOT NULL COMMENT 'name of the detail', + `value` varchar(255) NOT NULL COMMENT 'value of the detail', + PRIMARY KEY (`id`), + CONSTRAINT `fk_object_store_details__store_id` FOREIGN KEY `fk_object_store__store_id`(`store_id`) REFERENCES `object_store`(`id`) ON DELETE CASCADE, + INDEX `i_object_store__name__value`(`name`(128), `value`(128)) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`bucket`; +CREATE TABLE `cloud`.`bucket` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `name` varchar(255) NOT NULL COMMENT 'name of bucket', + `object_store_id` varchar(255) NOT NULL COMMENT 'id of object_store', + `state` varchar(255) NOT NULL COMMENT 'state of the bucket', + `uuid` varchar(255) COMMENT 'uuid of bucket', + `domain_id` bigint unsigned NOT NULL COMMENT 'domain the bucket belongs to', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + `quota` bigint unsigned COMMENT 'Allocated bucket quota in GB', + `versioning` boolean COMMENT 'versioning enable/disable', + `encryption` boolean COMMENT 'encryption enable/disbale', + `object_lock` boolean COMMENT 'Lock objects in bucket', + `policy` varchar(255) COMMENT 'Bucket Access Policy', + `access_key` varchar(255) COMMENT 'Bucket Access Key', + `secret_key` varchar(255) COMMENT 'Bucket Secret Key', + `bucket_url` varchar(255) COMMENT 'URL to access bucket', + `created` datetime COMMENT 'date the bucket was created', + `removed` datetime COMMENT 'date removed if not null', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud`.`bucket_statistics`; +CREATE TABLE `cloud`.`bucket_statistics` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `bucket_id` bigint unsigned NOT NULL COMMENT 'id of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +DROP TABLE IF EXISTS `cloud_usage`.`bucket_statistics`; +CREATE TABLE `cloud_usage`.`bucket_statistics` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT COMMENT 'id', + `account_id` bigint unsigned NOT NULL COMMENT 'owner of this bucket', + `bucket_id` bigint unsigned NOT NULL COMMENT 'id of this bucket', + `size` bigint unsigned COMMENT 'total size of bucket objects', + PRIMARY KEY(`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/engine/storage/integration-test/src/test/resources/storageContext.xml b/engine/storage/integration-test/src/test/resources/storageContext.xml index fc24753127e..7c95345f673 100644 --- a/engine/storage/integration-test/src/test/resources/storageContext.xml +++ b/engine/storage/integration-test/src/test/resources/storageContext.xml @@ -87,4 +87,8 @@ + + + + diff --git a/engine/storage/object/pom.xml b/engine/storage/object/pom.xml new file mode 100644 index 00000000000..5bad4e8a3d6 --- /dev/null +++ b/engine/storage/object/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + cloud-engine-storage-object + Apache CloudStack Engine Storage Object Component + + org.apache.cloudstack + cloud-engine + 4.19.0.0-SNAPSHOT + ../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java new file mode 100644 index 00000000000..418121503c1 --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/BucketObject.java @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.DataTO; +import org.apache.cloudstack.engine.subsystem.api.storage.BucketInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; + +import java.util.Date; + +public class BucketObject implements BucketInfo { + + private String name; + + @Override + public long getDomainId() { + return 0; + } + + @Override + public long getAccountId() { + return 0; + } + + @Override + public Class getEntityType() { + return null; + } + + @Override + public String getName() { + return this.name; + } + + @Override + public void setName(String name) { + this.name = name; + } + + @Override + public void addPayload(Object data) { + + } + + @Override + public Object getPayload() { + return null; + } + + @Override + public Bucket getBucket() { + return null; + } + + @Override + public long getId() { + return 0; + } + + @Override + public String getUri() { + return null; + } + + @Override + public DataTO getTO() { + return null; + } + + @Override + public DataStore getDataStore() { + return null; + } + + @Override + public Long getSize() { + return null; + } + + @Override + public Integer getQuota() { + return null; + } + + @Override + public boolean isVersioning() { + return false; + } + + @Override + public boolean isEncryption() { + return false; + } + + @Override + public boolean isObjectLock() { + return false; + } + + @Override + public String getPolicy() { + return null; + } + + @Override + public String getBucketURL() { + return null; + } + + @Override + public String getAccessKey() { + return null; + } + + @Override + public String getSecretKey() { + return null; + } + + @Override + public long getPhysicalSize() { + return 0; + } + + @Override + public DataObjectType getType() { + return null; + } + + @Override + public String getUuid() { + return null; + } + + @Override + public boolean delete() { + return false; + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event) { + + } + + @Override + public void processEvent(ObjectInDataStoreStateMachine.Event event, Answer answer) { + + } + + @Override + public void incRefCount() { + + } + + @Override + public void decRefCount() { + + } + + @Override + public Long getRefCount() { + return null; + } + + @Override + public long getObjectStoreId() { + return 0; + } + + @Override + public Date getCreated() { + return null; + } + + @Override + public State getState() { + return null; + } +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java new file mode 100644 index 00000000000..a0db89bad4e --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/ObjectStorageServiceImpl.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.storage.object; + +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStorageService; +import org.apache.log4j.Logger; + +public class ObjectStorageServiceImpl implements ObjectStorageService { + + private static final Logger s_logger = Logger.getLogger(ObjectStorageServiceImpl.class); + + +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java new file mode 100644 index 00000000000..40f503692e1 --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImpl.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.manager; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.ObjectStoreImpl; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Component +public class ObjectStoreProviderManagerImpl implements ObjectStoreProviderManager, Configurable { + private static final Logger s_logger = Logger.getLogger(ObjectStoreProviderManagerImpl.class); + @Inject + ObjectStoreDao objectStoreDao; + + @Inject + DataStoreProviderManager providerManager; + + Map driverMaps; + + @PostConstruct + public void config() { + driverMaps = new HashMap(); + } + + @Override + public ObjectStoreEntity getObjectStore(long objectStoreId) { + ObjectStoreVO objectStore = objectStoreDao.findById(objectStoreId); + String providerName = objectStore.getProviderName(); + ObjectStoreProvider provider = (ObjectStoreProvider)providerManager.getDataStoreProvider(providerName); + ObjectStoreEntity objStore = ObjectStoreImpl.getDataStore(objectStore, driverMaps.get(provider.getName()), provider); + return objStore; + } + + @Override + public boolean registerDriver(String providerName, ObjectStoreDriver driver) { + if (driverMaps.containsKey(providerName)) { + return false; + } + driverMaps.put(providerName, driver); + return true; + } + + @Override + public ObjectStoreEntity getObjectStore(String uuid) { + ObjectStoreVO objectStore = objectStoreDao.findByUuid(uuid); + return getObjectStore(objectStore.getId()); + } + + @Override + public List listObjectStores() { + List stores = objectStoreDao.listObjectStores(); + List ObjectStores = new ArrayList(); + for (ObjectStoreVO store : stores) { + ObjectStores.add(getObjectStore(store.getId())); + } + return ObjectStores; + } + + @Override + public List listObjectStoreByProvider(String provider) { + List stores = objectStoreDao.findByProvider(provider); + List ObjectStores = new ArrayList(); + for (ObjectStoreVO store : stores) { + ObjectStores.add(getObjectStore(store.getId())); + } + return ObjectStores; + } + + @Override + public String getConfigComponentName() { + return ObjectStoreProviderManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { }; + } +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java new file mode 100644 index 00000000000..825b349bdfc --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/ObjectStoreImpl.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.store; + +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.DataStoreRole; +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.log4j.Logger; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class ObjectStoreImpl implements ObjectStoreEntity { + private static final Logger s_logger = Logger.getLogger(ObjectStoreImpl.class); + + protected ObjectStoreDriver driver; + protected ObjectStoreVO objectStoreVO; + protected ObjectStoreProvider provider; + + public ObjectStoreImpl() { + super(); + } + + protected void configure(ObjectStoreVO objectStoreVO, ObjectStoreDriver objectStoreDriver, ObjectStoreProvider provider) { + this.driver = objectStoreDriver; + this.objectStoreVO = objectStoreVO; + this.provider = provider; + } + + public static ObjectStoreEntity getDataStore(ObjectStoreVO objectStoreVO, ObjectStoreDriver objectStoreDriver, ObjectStoreProvider provider) { + ObjectStoreImpl instance = ComponentContext.inject(ObjectStoreImpl.class); + instance.configure(objectStoreVO, objectStoreDriver, provider); + return instance; + } + + @Override + public DataStoreDriver getDriver() { + return this.driver; + } + + @Override + public DataStoreRole getRole() { + return null; + } + + @Override + public long getId() { + return this.objectStoreVO.getId(); + } + + @Override + public String getUri() { + return this.objectStoreVO.getUrl(); + } + + @Override + public Scope getScope() { + return null; + } + + + @Override + public String getUuid() { + return this.objectStoreVO.getUuid(); + } + + public Date getCreated() { + return this.objectStoreVO.getCreated(); + } + + @Override + public String getName() { + return objectStoreVO.getName(); + } + + @Override + public DataObject create(DataObject obj) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + return driver.createBucket(bucket, objectLock); + } + + @Override + public boolean deleteBucket(String bucketName) { + return driver.deleteBucket(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean setBucketEncryption(String bucketName) { + return driver.setBucketEncryption(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean deleteBucketEncryption(String bucketName) { + return driver.deleteBucketEncryption(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean setBucketVersioning(String bucketName) { + return driver.setBucketVersioning(bucketName, objectStoreVO.getId()); + } + + @Override + public boolean deleteBucketVersioning(String bucketName) { + return driver.deleteBucketVersioning(bucketName, objectStoreVO.getId()); + } + + @Override + public void setBucketPolicy(String bucketName, String policy) { + driver.setBucketPolicy(bucketName, policy, objectStoreVO.getId()); + } + + @Override + public void setQuota(String bucketName, int quota) { + driver.setBucketQuota(bucketName, objectStoreVO.getId(), quota); + } + + @Override + public Map getAllBucketsUsage() { + return driver.getAllBucketsUsage(objectStoreVO.getId()); + } + + @Override + public List listBuckets() { + return driver.listBuckets(objectStoreVO.getId()); + } + + /* + Create user if not exists + */ + @Override + public boolean createUser(long accountId) { + return driver.createUser(accountId, objectStoreVO.getId()); + } + + @Override + public boolean delete(DataObject obj) { + return false; + } + + @Override + public DataStoreTO getTO() { + return null; + } + + @Override + public String getProviderName() { + return objectStoreVO.getProviderName(); + } + + @Override + public String getUrl() { + return objectStoreVO.getUrl(); + } + +} diff --git a/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java new file mode 100644 index 00000000000..ff88262c4ba --- /dev/null +++ b/engine/storage/object/src/main/java/org/apache/cloudstack/storage/object/store/lifecycle/ObjectStoreLifeCycle.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.store.lifecycle; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; + +public interface ObjectStoreLifeCycle extends DataStoreLifeCycle { +} diff --git a/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml b/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml new file mode 100644 index 00000000000..57bd9f87749 --- /dev/null +++ b/engine/storage/object/src/main/resources/META-INF/cloudstack/core/spring-engine-storage-object-core-context.xml @@ -0,0 +1,36 @@ + + + + + + + + diff --git a/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java b/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java new file mode 100644 index 00000000000..392388ca8b8 --- /dev/null +++ b/engine/storage/object/src/test/java/org/apache/cloudstack/storage/object/manager/ObjectStoreProviderManagerImplTest.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.object.manager; + +import junit.framework.TestCase; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; +import org.apache.cloudstack.storage.object.store.ObjectStoreImpl; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class ObjectStoreProviderManagerImplTest extends TestCase{ + + ObjectStoreProviderManagerImpl objectStoreProviderManagerImplSpy; + + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + ObjectStoreVO objectStoreVO; + + @Mock + DataStoreProviderManager providerManager; + + @Mock + ObjectStoreProvider provider; + + @Mock + Map driverMaps; + + @Mock + ObjectStoreDriver objectStoreDriver; + + @Mock + ObjectStoreEntity objectStoreEntity; + + MockedStatic mockObjectStoreImpl; + + @Before + public void setup(){ + objectStoreProviderManagerImplSpy = Mockito.spy(new ObjectStoreProviderManagerImpl()); + objectStoreProviderManagerImplSpy.objectStoreDao = objectStoreDao; + objectStoreProviderManagerImplSpy.providerManager = providerManager; + objectStoreProviderManagerImplSpy.driverMaps = driverMaps; + mockObjectStoreImpl = mockStatic(ObjectStoreImpl.class); + } + + @Test + public void getObjectStoreTest() { + Mockito.doReturn(objectStoreVO).when(objectStoreDao).findById(Mockito.anyLong()); + Mockito.doReturn(provider).when(providerManager).getDataStoreProvider(Mockito.anyString()); + Mockito.doReturn(objectStoreDriver).when(driverMaps).get(Mockito.anyString()); + Mockito.doReturn("Simulator").when(provider).getName(); + Mockito.doReturn("Simulator").when(objectStoreVO).getProviderName(); + + when(ObjectStoreImpl.getDataStore(Mockito.any(ObjectStoreVO.class), Mockito.any(ObjectStoreDriver.class), + Mockito.any(ObjectStoreProvider.class))).thenReturn(objectStoreEntity); + assertNotNull(objectStoreProviderManagerImplSpy.getObjectStore(1L)); + } + + @Test + public void listObjectStoresTest() { + List stores = new ArrayList<>(); + stores.add(objectStoreVO); + Mockito.doReturn(objectStoreVO).when(objectStoreDao).findById(Mockito.anyLong()); + Mockito.doReturn(provider).when(providerManager).getDataStoreProvider(Mockito.anyString()); + Mockito.doReturn(objectStoreDriver).when(driverMaps).get(Mockito.anyString()); + Mockito.doReturn("Simulator").when(provider).getName(); + Mockito.doReturn("Simulator").when(objectStoreVO).getProviderName(); + when(ObjectStoreImpl.getDataStore(Mockito.any(ObjectStoreVO.class), Mockito.any(ObjectStoreDriver.class), + Mockito.any(ObjectStoreProvider.class))).thenReturn(objectStoreEntity); + Mockito.doReturn(stores).when(objectStoreDao).listObjectStores(); + assertEquals(1, objectStoreProviderManagerImplSpy.listObjectStores().size()); + } + + @Override + @After + public void tearDown() throws Exception { + mockObjectStoreImpl.close(); + super.tearDown(); + } +} diff --git a/engine/storage/object/src/test/resource/testContext.xml b/engine/storage/object/src/test/resource/testContext.xml new file mode 100644 index 00000000000..7352b1148f7 --- /dev/null +++ b/engine/storage/object/src/test/resource/testContext.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.cloudstack.framework + + + + + + + + + + + + + + + + + + + + + + diff --git a/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/engine/storage/object/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index cd525ae0ef7..60d01119302 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.engine.subsystem.api.storage.Scope; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.springframework.stereotype.Component; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; @@ -40,6 +41,8 @@ public class DataStoreManagerImpl implements DataStoreManager { PrimaryDataStoreProviderManager primaryStoreMgr; @Inject ImageStoreProviderManager imageDataStoreMgr; + @Inject + ObjectStoreProviderManager objectStoreProviderMgr; @Override public DataStore getDataStore(long storeId, DataStoreRole role) { @@ -50,6 +53,8 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.getImageStore(storeId); } else if (role == DataStoreRole.ImageCache) { return imageDataStoreMgr.getImageStore(storeId); + } else if (role == DataStoreRole.Object) { + return objectStoreProviderMgr.getObjectStore(storeId); } } catch (CloudRuntimeException e) { throw e; @@ -63,6 +68,8 @@ public class DataStoreManagerImpl implements DataStoreManager { return primaryStoreMgr.getPrimaryDataStore(uuid); } else if (role == DataStoreRole.Image) { return imageDataStoreMgr.getImageStore(uuid); + } else if (role == DataStoreRole.Object) { + return objectStoreProviderMgr.getObjectStore(uuid); } throw new CloudRuntimeException("un recognized type" + role); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java index 98eeb6b4405..35e758accca 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/provider/DataStoreProviderManagerImpl.java @@ -30,6 +30,8 @@ import java.util.concurrent.CopyOnWriteArrayList; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -56,6 +58,8 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto PrimaryDataStoreProviderManager primaryDataStoreProviderMgr; @Inject ImageStoreProviderManager imageStoreProviderMgr; + @Inject + ObjectStoreProviderManager objectStoreProviderMgr; @Override public DataStoreProvider getDataStoreProvider(String name) { @@ -144,6 +148,8 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto primaryDataStoreProviderMgr.registerHostListener(provider.getName(), provider.getHostListener()); } else if (types.contains(DataStoreProviderType.IMAGE)) { imageStoreProviderMgr.registerDriver(provider.getName(), (ImageStoreDriver)provider.getDataStoreDriver()); + } else if (types.contains(DataStoreProviderType.OBJECT)) { + objectStoreProviderMgr.registerDriver(provider.getName(), (ObjectStoreDriver)provider.getDataStoreDriver()); } } catch (Exception e) { s_logger.debug("configure provider failed", e); @@ -169,6 +175,11 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return this.getDataStoreProvider(DataStoreProvider.NFS_IMAGE); } + @Override + public DataStoreProvider getDefaultObjectStoreProvider() { + return this.getDataStoreProvider(DataStoreProvider.S3_IMAGE); + } + @Override public List getDataStoreProviders(String type) { if (type == null) { @@ -180,7 +191,9 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return this.getImageDataStoreProviders(); } else if (type.equalsIgnoreCase(DataStoreProvider.DataStoreProviderType.ImageCache.toString())) { return this.getCacheDataStoreProviders(); - } else { + } else if (type.equalsIgnoreCase(DataStoreProviderType.OBJECT.toString())) { + return this.getObjectStoreProviders(); + }else { throw new InvalidParameterValueException("Invalid parameter: " + type); } } @@ -223,4 +236,16 @@ public class DataStoreProviderManagerImpl extends ManagerBase implements DataSto return providers; } + public List getObjectStoreProviders() { + List providers = new ArrayList(); + for (DataStoreProvider provider : providerMap.values()) { + if (provider.getTypes().contains(DataStoreProviderType.OBJECT)) { + StorageProviderResponse response = new StorageProviderResponse(); + response.setName(provider.getName()); + response.setType(DataStoreProviderType.OBJECT.toString()); + providers.add(response); + } + } + return providers; + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java new file mode 100644 index 00000000000..e6027a1959f --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/BaseObjectStoreDriverImpl.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.cloud.agent.api.to.DataTO; +import com.cloud.host.Host; +import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; +import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; +import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.framework.async.AsyncCompletionCallback; +import org.apache.cloudstack.framework.async.AsyncRpcContext; +import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.log4j.Logger; + +import java.util.Map; + +public abstract class BaseObjectStoreDriverImpl implements ObjectStoreDriver { + private static final Logger LOGGER = Logger.getLogger(BaseObjectStoreDriverImpl.class); + + @Override + public Map getCapabilities() { + return null; + } + + @Override + public DataTO getTO(DataObject data) { + return null; + } + + protected class CreateContext extends AsyncRpcContext { + final DataObject data; + + public CreateContext(AsyncCompletionCallback callback, DataObject data) { + super(callback); + this.data = data; + } + } + + @Override + public void createAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + } + + @Override + public void deleteAsync(DataStore dataStore, DataObject data, AsyncCompletionCallback callback) { + } + + @Override + public void copyAsync(DataObject srcdata, DataObject destData, AsyncCompletionCallback callback) { + } + + @Override + public void copyAsync(DataObject srcData, DataObject destData, Host destHost, AsyncCompletionCallback callback) { + copyAsync(srcData, destData, callback); + } + + @Override + public boolean canCopy(DataObject srcData, DataObject destData) { + return false; + } + + @Override + public void resize(DataObject data, AsyncCompletionCallback callback) { + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java new file mode 100644 index 00000000000..4953b9b0cdf --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/ObjectStoreDriver.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; + +import java.util.List; +import java.util.Map; + +public interface ObjectStoreDriver extends DataStoreDriver { + Bucket createBucket(Bucket bucket, boolean objectLock); + + List listBuckets(long storeId); + + boolean deleteBucket(String bucketName, long storeId); + + AccessControlList getBucketAcl(String bucketName, long storeId); + + void setBucketAcl(String bucketName, AccessControlList acl, long storeId); + + void setBucketPolicy(String bucketName, String policyType, long storeId); + + BucketPolicy getBucketPolicy(String bucketName, long storeId); + + void deleteBucketPolicy(String bucketName, long storeId); + + boolean createUser(long accountId, long storeId); + + boolean setBucketEncryption(String bucketName, long storeId); + + boolean deleteBucketEncryption(String bucketName, long storeId); + + + boolean setBucketVersioning(String bucketName, long storeId); + + boolean deleteBucketVersioning(String bucketName, long storeId); + + void setBucketQuota(String bucketName, long storeId, long size); + + Map getAllBucketsUsage(long storeId); +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java new file mode 100644 index 00000000000..c58d801e40e --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreHelper.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.datastore; + +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailVO; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; + +@Component +public class ObjectStoreHelper { + @Inject + ObjectStoreDao ObjectStoreDao; + @Inject + ObjectStoreDetailsDao ObjectStoreDetailsDao; + + public ObjectStoreVO createObjectStore(Map params, Map details) { + ObjectStoreVO store = new ObjectStoreVO(); + + store.setProviderName((String)params.get("providerName")); + store.setUuid(UUID.randomUUID().toString()); + store.setUrl((String)params.get("url")); + store.setName((String)params.get("name")); + + store = ObjectStoreDao.persist(store); + + // persist details + if (details != null) { + Iterator keyIter = details.keySet().iterator(); + while (keyIter.hasNext()) { + String key = keyIter.next().toString(); + String value = details.get(key); + ObjectStoreDetailVO detail = new ObjectStoreDetailVO(store.getId(), key, value); + ObjectStoreDetailsDao.persist(detail); + } + } + return store; + } + + public boolean deleteObjectStore(long id) { + ObjectStoreVO store = ObjectStoreDao.findById(id); + if (store == null) { + throw new CloudRuntimeException("can't find Object store:" + id); + } + + ObjectStoreDao.remove(id); + return true; + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java new file mode 100644 index 00000000000..b23f3194139 --- /dev/null +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/object/datastore/ObjectStoreProviderManager.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.object.datastore; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; + +import java.util.List; + +public interface ObjectStoreProviderManager { + ObjectStoreEntity getObjectStore(String uuid); + + List listObjectStores(); + + List listObjectStoreByProvider(String provider); + + ObjectStoreEntity getObjectStore(long objectStoreId); + + boolean registerDriver(String uuid, ObjectStoreDriver driver); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 553dc840c0d..6f19fa95f1b 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -55,6 +55,7 @@ public class QuotaTypes extends UsageTypes { quotaTypeList.put(VOLUME_SECONDARY, new QuotaTypes(VOLUME_SECONDARY, "VOLUME_SECONDARY", UsageUnitTypes.GB_MONTH.toString(), "Volume secondary storage usage")); quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage")); quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage")); + quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage")); quotaTypeMap = Collections.unmodifiableMap(quotaTypeList); } diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java index a27896aa58d..51c020fc237 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/metrics/MetricsServiceImpl.java @@ -71,6 +71,7 @@ import org.apache.cloudstack.response.VolumeMetricsResponse; import org.apache.cloudstack.response.VolumeMetricsStatsResponse; import org.apache.cloudstack.response.ZoneMetricsResponse; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; import org.apache.commons.beanutils.BeanUtils; @@ -176,6 +177,9 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements @Inject private VolumeStatsDao volumeStatsDao; + @Inject + private ObjectStoreDao objectStoreDao; + private static Gson gson = new Gson(); protected MetricsServiceImpl() { @@ -557,6 +561,7 @@ public class MetricsServiceImpl extends MutualExclusiveIdsManagerBase implements response.setHosts(hostDao.countAllByType(Host.Type.Routing)); response.setStoragePools(storagePoolDao.countAll()); response.setImageStores(imageStoreDao.countAllImageStores()); + response.setObjectStores(objectStoreDao.countAllObjectStores()); response.setSystemvms(vmInstanceDao.listByTypes(VirtualMachine.Type.ConsoleProxy, VirtualMachine.Type.SecondaryStorageVm).size()); response.setRouters(domainRouterDao.countAllByRole(VirtualRouter.Role.VIRTUAL_ROUTER)); response.setInternalLbs(domainRouterDao.countAllByRole(VirtualRouter.Role.INTERNAL_LB_VM)); diff --git a/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java b/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java index 280b7998fbb..cb1faf237b7 100644 --- a/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java +++ b/plugins/metrics/src/main/java/org/apache/cloudstack/response/InfrastructureResponse.java @@ -47,6 +47,10 @@ public class InfrastructureResponse extends BaseResponse { @Param(description = "Number of images stores") private Integer imageStores; + @SerializedName("objectstores") + @Param(description = "Number of object stores") + private Integer objectStores; + @SerializedName("systemvms") @Param(description = "Number of systemvms") private Integer systemvms; @@ -118,4 +122,8 @@ public class InfrastructureResponse extends BaseResponse { public void setAlerts(Integer alerts) { this.alerts = alerts; } public void setInternalLbs(Integer internalLbs) { this.internalLbs = internalLbs; } + + public void setObjectStores(Integer objectStores) { + this.objectStores = objectStores; + } } diff --git a/plugins/pom.xml b/plugins/pom.xml index 06c5fab6a12..6c4d561f896 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -133,6 +133,8 @@ storage/volume/scaleio storage/volume/linstor storage/volume/storpool + storage/object/minio + storage/object/simulator storage-allocators/random @@ -187,6 +189,30 @@ test-jar test + + org.apache.cloudstack + cloud-engine-storage + 4.19.0.0-SNAPSHOT + compile + + + io.minio + minio + 8.5.2 + compile + + + io.minio + minio-admin + 8.5.2 + compile + + + org.apache.cloudstack + cloud-engine-storage-object + 4.19.0.0-SNAPSHOT + compile + diff --git a/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java b/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java index 19ded7844db..9f75251c93f 100644 --- a/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java +++ b/plugins/shutdown/src/test/java/org/apache/cloudstack/shutdown/ShutdownManagerImplTest.java @@ -18,12 +18,15 @@ package org.apache.cloudstack.shutdown; import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -39,6 +42,12 @@ public class ShutdownManagerImplTest { @Mock AsyncJobManager jobManagerMock; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } private long prepareCountPendingJobs() { long expectedCount = 1L; @@ -75,4 +84,9 @@ public class ShutdownManagerImplTest { spy.cancelShutdown(); Mockito.verify(jobManagerMock).enableAsyncJobs(); } + + @After + public void tearDown() throws Exception { + closeable.close(); + } } diff --git a/plugins/storage/object/minio/pom.xml b/plugins/storage/object/minio/pom.xml new file mode 100644 index 00000000000..704772a8ab4 --- /dev/null +++ b/plugins/storage/object/minio/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + cloud-plugin-storage-object-minio + Apache CloudStack Plugin - MinIO object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.19.0.0-SNAPSHOT + ../../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + io.minio + minio + 8.5.2 + + + io.minio + minio-admin + 8.5.2 + + + diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java new file mode 100644 index 00000000000..15df5df56dc --- /dev/null +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImpl.java @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.Account; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import io.minio.BucketExistsArgs; +import io.minio.DeleteBucketEncryptionArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.RemoveBucketArgs; +import io.minio.SetBucketEncryptionArgs; +import io.minio.SetBucketPolicyArgs; +import io.minio.SetBucketVersioningArgs; +import io.minio.admin.MinioAdminClient; +import io.minio.admin.QuotaUnit; +import io.minio.admin.UserInfo; +import io.minio.admin.messages.DataUsageInfo; +import io.minio.messages.SseConfiguration; +import io.minio.messages.VersioningConfiguration; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.cloudstack.storage.object.BucketObject; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.inject.Inject; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class MinIOObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = Logger.getLogger(MinIOObjectStoreDriverImpl.class); + + @Inject + AccountDao _accountDao; + + @Inject + AccountDetailsDao _accountDetailsDao; + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + @Inject + ObjectStoreDetailsDao _storeDetailsDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + private static final String MINIO_ACCESS_KEY = "minio-accesskey"; + private static final String MINIO_SECRET_KEY = "minio-secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + //ToDo Client pool mgmt + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + long accountId = bucket.getAccountId(); + MinioClient minioClient = getMinIOClient(storeId); + Account account = _accountDao.findById(accountId); + + if ((_accountDetailsDao.findDetail(accountId, MINIO_ACCESS_KEY) == null) + || (_accountDetailsDao.findDetail(accountId, MINIO_SECRET_KEY) == null)) { + throw new CloudRuntimeException("Bucket access credentials unavailable for account: "+account.getAccountName()); + } + + try { + if(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket already exists with name "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + try { + minioClient.makeBucket(MakeBucketArgs.builder().bucket(bucketName).objectLock(objectLock).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + + List buckets = _bucketDao.listByObjectStoreIdAndAccountId(storeId, accountId); + StringBuilder resources_builder = new StringBuilder(); + for(BucketVO exitingBucket : buckets) { + resources_builder.append("\"arn:aws:s3:::"+exitingBucket.getName()+"/*\",\n"); + } + resources_builder.append("\"arn:aws:s3:::"+bucketName+"/*\"\n"); + + String policy = " {\n" + + " \"Statement\": [\n" + + " {\n" + + " \"Action\": \"s3:*\",\n" + + " \"Effect\": \"Allow\",\n" + + " \"Principal\": \"*\",\n" + + " \"Resource\": ["+resources_builder+"]" + + " }\n" + + " ],\n" + + " \"Version\": \"2012-10-17\"\n" + + " }"; + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String policyName = "acs-"+account.getAccountName()+"-policy"; + String userName = "acs-"+account.getAccountName(); + try { + minioAdminClient.addCannedPolicy(policyName, policy); + minioAdminClient.setPolicy(userName, false, policyName); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + String accessKey = _accountDetailsDao.findDetail(accountId, MINIO_ACCESS_KEY).getValue(); + String secretKey = _accountDetailsDao.findDetail(accountId, MINIO_SECRET_KEY).getValue(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(accessKey); + bucketVO.setSecretKey(secretKey); + bucketVO.setBucketURL(store.getUrl()+"/"+bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } + + @Override + public List listBuckets(long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + List bucketsList = new ArrayList<>(); + try { + List minIOBuckets = minioClient.listBuckets(); + for(io.minio.messages.Bucket minIObucket : minIOBuckets) { + Bucket bucket = new BucketObject(); + bucket.setName(minIObucket.name()); + bucketsList.add(bucket); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + if(!minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())) { + throw new CloudRuntimeException("Bucket doesn't exist: "+ bucketName); + } + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + //ToDo: check bucket empty + try { + minioClient.removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + return null; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + String privatePolicy = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"; + + StringBuilder builder = new StringBuilder(); + builder.append("{\n"); + builder.append(" \"Statement\": [\n"); + builder.append(" {\n"); + builder.append(" \"Action\": [\n"); + builder.append(" \"s3:GetBucketLocation\",\n"); + builder.append(" \"s3:ListBucket\"\n"); + builder.append(" ],\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"\"\n"); + builder.append(" },\n"); + builder.append(" {\n"); + builder.append(" \"Action\": \"s3:GetObject\",\n"); + builder.append(" \"Effect\": \"Allow\",\n"); + builder.append(" \"Principal\": \"*\",\n"); + builder.append(" \"Resource\": \"arn:aws:s3:::"+bucketName+"/*\"\n"); + builder.append(" }\n"); + builder.append(" ],\n"); + builder.append(" \"Version\": \"2012-10-17\"\n"); + builder.append("}\n"); + + String publicPolicy = builder.toString(); + + //ToDo Support custom policy + String policyConfig = (policy.equalsIgnoreCase("public"))? publicPolicy : privatePolicy; + + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketPolicy( + SetBucketPolicyArgs.builder().bucket(bucketName).config(policyConfig).build()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + return null; + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + + } + + @Override + public boolean createUser(long accountId, long storeId) { + Account account = _accountDao.findById(accountId); + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + String accessKey = "acs-"+account.getAccountName(); + // Check user exists + try { + UserInfo userInfo = minioAdminClient.getUserInfo(accessKey); + if(userInfo != null) { + s_logger.debug("User already exists in MinIO store: "+accessKey); + return true; + } + } catch (Exception e) { + s_logger.debug("User does not exist. Creating user: "+accessKey); + } + + KeyGenerator generator = null; + try { + generator = KeyGenerator.getInstance("HmacSHA1"); + } catch (NoSuchAlgorithmException e) { + throw new CloudRuntimeException(e); + } + SecretKey key = generator.generateKey(); + String secretKey = Base64.encodeBase64URLSafeString(key.getEncoded()); + try { + minioAdminClient.addUser(accessKey, UserInfo.Status.ENABLED, secretKey, "", new ArrayList()); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + // Store user credentials + Map details = new HashMap<>(); + details.put(MINIO_ACCESS_KEY, accessKey); + details.put(MINIO_SECRET_KEY, secretKey); + _accountDetailsDao.persist(accountId, details); + return true; + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketEncryption(SetBucketEncryptionArgs.builder() + .bucket(bucketName) + .config(SseConfiguration.newConfigWithSseS3Rule()) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.deleteBucketEncryption(DeleteBucketEncryptionArgs.builder() + .bucket(bucketName) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.ENABLED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + MinioClient minioClient = getMinIOClient(storeId); + try { + minioClient.setBucketVersioning(SetBucketVersioningArgs.builder() + .bucket(bucketName) + .config(new VersioningConfiguration(VersioningConfiguration.Status.SUSPENDED, null)) + .build() + ); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + return true; + } + + @Override + public void setBucketQuota(String bucketName, long storeId, long size) { + + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + try { + minioAdminClient.setBucketQuota(bucketName, size, QuotaUnit.GB); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + @Override + public Map getAllBucketsUsage(long storeId) { + MinioAdminClient minioAdminClient = getMinIOAdminClient(storeId); + try { + DataUsageInfo dataUsageInfo = minioAdminClient.getDataUsageInfo(); + return dataUsageInfo.bucketsSizes(); + } catch (Exception e) { + throw new CloudRuntimeException(e); + } + } + + protected MinioClient getMinIOClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String url = store.getUrl(); + String accessKey = storeDetails.get(ACCESS_KEY); + String secretKey = storeDetails.get(SECRET_KEY); + MinioClient minioClient = + MinioClient.builder() + .endpoint(url) + .credentials(accessKey,secretKey) + .build(); + if(minioClient == null){ + throw new CloudRuntimeException("Error while creating MinIO client"); + } + return minioClient; + } + + protected MinioAdminClient getMinIOAdminClient(long storeId) { + ObjectStoreVO store = _storeDao.findById(storeId); + Map storeDetails = _storeDetailsDao.getDetails(storeId); + String url = store.getUrl(); + String accessKey = storeDetails.get(ACCESS_KEY); + String secretKey = storeDetails.get(SECRET_KEY); + MinioAdminClient minioAdminClient = + MinioAdminClient.builder() + .endpoint(url) + .credentials(accessKey,secretKey) + .build(); + if(minioAdminClient == null){ + throw new CloudRuntimeException("Error while creating MinIO client"); + } + return minioAdminClient; + } +} diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java new file mode 100644 index 00000000000..fb7d1a652fc --- /dev/null +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/MinIOObjectStoreLifeCycleImpl.java @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.utils.exception.CloudRuntimeException; +import io.minio.MinioClient; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.Map; + +public class MinIOObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger s_logger = Logger.getLogger(MinIOObjectStoreLifeCycleImpl.class); + + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + public MinIOObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + + String url = (String)dsInfos.get("url"); + String name = (String)dsInfos.get("name"); + String providerName = (String)dsInfos.get("providerName"); + Map details = (Map)dsInfos.get("details"); + if(details == null){ + throw new CloudRuntimeException("MinIO credentials are missing"); + } + String accessKey = details.get("accesskey"); + String secretKey = details.get("secretkey"); + + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + + objectStoreParameters.put("providerName", providerName); + objectStoreParameters.put("accesskey", accessKey); + objectStoreParameters.put("secretkey", secretKey); + + //check credentials + MinioClient minioClient = + MinioClient.builder() + .endpoint(url) + .credentials(accessKey,secretKey) + .build(); + try { + // Test connection by listing buckets + minioClient.listBuckets(); + s_logger.debug("Successfully connected to MinIO EndPoint: "+url); + } catch (Exception e) { + s_logger.debug("Error while initializing MinIO Object Store: "+e.getMessage()); + throw new RuntimeException("Error while initializing MinIO Object Store. Invalid credentials or URL"); + } + + ObjectStoreVO objectStore = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(objectStore.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java new file mode 100644 index 00000000000..abcc4996b88 --- /dev/null +++ b/plugins/storage/object/minio/src/main/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.MinIOObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.MinIOObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class MinIOObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + @Inject + ObjectStoreHelper helper; + + private final String providerName = "MinIO"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(MinIOObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(MinIOObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties new file mode 100644 index 00000000000..828454e4634 --- /dev/null +++ b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=storage-object-minio +parent=storage diff --git a/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml new file mode 100644 index 00000000000..30876de2e0a --- /dev/null +++ b/plugins/storage/object/minio/src/main/resources/META-INF/cloudstack/storage-object-minio/spring-storage-object-minio-context.xml @@ -0,0 +1,31 @@ + + + + diff --git a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java new file mode 100644 index 00000000000..3041a5b232b --- /dev/null +++ b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/driver/MinIOObjectStoreDriverImplTest.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.driver; + +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import com.cloud.user.AccountDetailVO; +import com.cloud.user.AccountDetailsDao; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import io.minio.BucketExistsArgs; +import io.minio.MinioClient; +import io.minio.RemoveBucketArgs; +import io.minio.admin.MinioAdminClient; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.ArrayList; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyLong; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class MinIOObjectStoreDriverImplTest { + + @Spy + MinIOObjectStoreDriverImpl minioObjectStoreDriverImpl = new MinIOObjectStoreDriverImpl(); + + @Mock + MinioClient minioClient; + @Mock + MinioAdminClient minioAdminClient; + @Mock + ObjectStoreDao objectStoreDao; + @Mock + ObjectStoreVO objectStoreVO; + @Mock + ObjectStoreDetailsDao objectStoreDetailsDao; + @Mock + AccountDao accountDao; + @Mock + BucketDao bucketDao; + @Mock + AccountVO account; + @Mock + AccountDetailsDao accountDetailsDao; + + Bucket bucket; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + minioObjectStoreDriverImpl._storeDao = objectStoreDao; + minioObjectStoreDriverImpl._storeDetailsDao = objectStoreDetailsDao; + minioObjectStoreDriverImpl._accountDao = accountDao; + minioObjectStoreDriverImpl._bucketDao = bucketDao; + minioObjectStoreDriverImpl._accountDetailsDao = accountDetailsDao; + bucket = new BucketVO(); + bucket.setName("test-bucket"); + when(objectStoreVO.getUrl()).thenReturn("http://localhost:9000"); + when(objectStoreDao.findById(any())).thenReturn(objectStoreVO); + } + + @Test + public void testCreateBucket() throws Exception { + doReturn(minioClient).when(minioObjectStoreDriverImpl).getMinIOClient(anyLong()); + doReturn(minioAdminClient).when(minioObjectStoreDriverImpl).getMinIOAdminClient(anyLong()); + when(bucketDao.listByObjectStoreIdAndAccountId(anyLong(), anyLong())).thenReturn(new ArrayList()); + when(account.getAccountName()).thenReturn("admin"); + when(accountDao.findById(anyLong())).thenReturn(account); + when(accountDetailsDao.findDetail(anyLong(),anyString())). + thenReturn(new AccountDetailVO(1L, "abc","def")); + when(bucketDao.findById(anyLong())).thenReturn(new BucketVO()); + Bucket bucketRet = minioObjectStoreDriverImpl.createBucket(bucket, false); + assertEquals(bucketRet.getName(), bucket.getName()); + verify(minioClient, times(1)).bucketExists(any()); + verify(minioClient, times(1)).makeBucket(any()); + } + + @Test + public void testDeleteBucket() throws Exception { + String bucketName = "test-bucket"; + doReturn(minioClient).when(minioObjectStoreDriverImpl).getMinIOClient(anyLong()); + when(minioClient.bucketExists(BucketExistsArgs.builder().bucket(bucketName).build())).thenReturn(true); + doNothing().when(minioClient).removeBucket(RemoveBucketArgs.builder().bucket(bucketName).build()); + boolean success = minioObjectStoreDriverImpl.deleteBucket(bucketName, 1L); + assertTrue(success); + verify(minioClient, times(1)).bucketExists(any()); + verify(minioClient, times(1)).removeBucket(any()); + } +} diff --git a/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java new file mode 100644 index 00000000000..8651e00b6ad --- /dev/null +++ b/plugins/storage/object/minio/src/test/java/org/apache/cloudstack/storage/datastore/provider/MinIOObjectStoreProviderImplTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider.DataStoreProviderType; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class MinIOObjectStoreProviderImplTest { + + private MinIOObjectStoreProviderImpl minioObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + minioObjectStoreProviderImpl = new MinIOObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = minioObjectStoreProviderImpl.getName(); + assertEquals("MinIO", name); + } + + @Test + public void testGetTypes() { + Set types = minioObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/plugins/storage/object/simulator/pom.xml b/plugins/storage/object/simulator/pom.xml new file mode 100644 index 00000000000..36ba0755fe9 --- /dev/null +++ b/plugins/storage/object/simulator/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + cloud-plugin-storage-object-simulator + Apache CloudStack Plugin - Simulator object storage provider + + org.apache.cloudstack + cloudstack-plugins + 4.19.0.0-SNAPSHOT + ../../../pom.xml + + + + org.apache.cloudstack + cloud-engine-storage + ${project.version} + + + org.apache.cloudstack + cloud-engine-storage-object + ${project.version} + + + org.apache.cloudstack + cloud-engine-schema + ${project.version} + + + io.minio + minio + 8.5.2 + + + io.minio + minio-admin + 8.5.2 + + + diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java new file mode 100644 index 00000000000..5f25a6061a7 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/driver/SimulatorObjectStoreDriverImpl.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.driver; + +import com.amazonaws.services.s3.model.AccessControlList; +import com.amazonaws.services.s3.model.BucketPolicy; +import com.cloud.agent.api.to.DataStoreTO; +import org.apache.cloudstack.storage.object.Bucket; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.BaseObjectStoreDriverImpl; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SimulatorObjectStoreDriverImpl extends BaseObjectStoreDriverImpl { + private static final Logger s_logger = Logger.getLogger(SimulatorObjectStoreDriverImpl.class); + + @Inject + ObjectStoreDao _storeDao; + + @Inject + BucketDao _bucketDao; + + private static final String ACCESS_KEY = "accesskey"; + private static final String SECRET_KEY = "secretkey"; + + @Override + public DataStoreTO getStoreTO(DataStore store) { + return null; + } + + @Override + public Bucket createBucket(Bucket bucket, boolean objectLock) { + String bucketName = bucket.getName(); + long storeId = bucket.getObjectStoreId(); + ObjectStoreVO store = _storeDao.findById(storeId); + BucketVO bucketVO = _bucketDao.findById(bucket.getId()); + bucketVO.setAccessKey(ACCESS_KEY); + bucketVO.setSecretKey(SECRET_KEY); + bucketVO.setBucketURL(store.getUrl()+"/"+bucketName); + _bucketDao.update(bucket.getId(), bucketVO); + return bucket; + } + + @Override + public List listBuckets(long storeId) { + List bucketsList = new ArrayList<>(); + return bucketsList; + } + + @Override + public boolean deleteBucket(String bucketName, long storeId) { + return true; + } + + @Override + public AccessControlList getBucketAcl(String bucketName, long storeId) { + return null; + } + + @Override + public void setBucketAcl(String bucketName, AccessControlList acl, long storeId) { + + } + + @Override + public void setBucketPolicy(String bucketName, String policy, long storeId) { + + } + + @Override + public BucketPolicy getBucketPolicy(String bucketName, long storeId) { + return null; + } + + @Override + public void deleteBucketPolicy(String bucketName, long storeId) { + + } + + @Override + public boolean createUser(long accountId, long storeId) { + return true; + } + + @Override + public boolean setBucketEncryption(String bucketName, long storeId) { + return true; + } + + @Override + public boolean deleteBucketEncryption(String bucketName, long storeId) { + return true; + } + + @Override + public boolean setBucketVersioning(String bucketName, long storeId) { + return true; + } + + @Override + public boolean deleteBucketVersioning(String bucketName, long storeId) { + return true; + } + + @Override + public void setBucketQuota(String bucketName, long storeId, long size) { + + } + + @Override + public Map getAllBucketsUsage(long storeId) { + return new HashMap(); + } +} diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java new file mode 100644 index 00000000000..34e928ced30 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorObjectStoreLifeCycleImpl.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.lifecycle; + +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.Discoverer; +import com.cloud.resource.ResourceManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SimulatorObjectStoreLifeCycleImpl implements ObjectStoreLifeCycle { + + private static final Logger s_logger = Logger.getLogger(SimulatorObjectStoreLifeCycleImpl.class); + @Inject + protected ResourceManager _resourceMgr; + @Inject + protected ObjectStoreDao objectStoreDao; + @Inject + ObjectStoreHelper objectStoreHelper; + @Inject + ObjectStoreProviderManager objectStoreMgr; + + protected List _discoverers; + + public List getDiscoverers() { + return _discoverers; + } + + public void setDiscoverers(List discoverers) { + this._discoverers = discoverers; + } + + public SimulatorObjectStoreLifeCycleImpl() { + } + + @SuppressWarnings("unchecked") + @Override + public DataStore initialize(Map dsInfos) { + + String url = (String)dsInfos.get("url"); + String name = (String)dsInfos.get("name"); + String providerName = (String)dsInfos.get("providerName"); + Map details = (Map)dsInfos.get("details"); + + Map objectStoreParameters = new HashMap(); + objectStoreParameters.put("name", name); + objectStoreParameters.put("url", url); + objectStoreParameters.put("providerName", providerName); + + ObjectStoreVO ids = objectStoreHelper.createObjectStore(objectStoreParameters, details); + return objectStoreMgr.getObjectStore(ids.getId()); + } + + @Override + public boolean attachCluster(DataStore store, ClusterScope scope) { + return false; + } + + @Override + public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { + return false; + } + + @Override + public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { + return false; + } + + @Override + public boolean maintain(DataStore store) { + return false; + } + + @Override + public boolean cancelMaintain(DataStore store) { + return false; + } + + @Override + public boolean deleteDataStore(DataStore store) { + return false; + } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + +} diff --git a/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java new file mode 100644 index 00000000000..651fac4d1f1 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImpl.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.provider; + +import com.cloud.utils.component.ComponentContext; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectStoreProvider; +import org.apache.cloudstack.storage.datastore.driver.SimulatorObjectStoreDriverImpl; +import org.apache.cloudstack.storage.datastore.lifecycle.SimulatorObjectStoreLifeCycleImpl; +import org.apache.cloudstack.storage.object.ObjectStoreDriver; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreHelper; +import org.apache.cloudstack.storage.object.datastore.ObjectStoreProviderManager; +import org.apache.cloudstack.storage.object.store.lifecycle.ObjectStoreLifeCycle; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +@Component +public class SimulatorObjectStoreProviderImpl implements ObjectStoreProvider { + + @Inject + ObjectStoreProviderManager storeMgr; + @Inject + ObjectStoreHelper helper; + + private final String providerName = "Simulator"; + protected ObjectStoreLifeCycle lifeCycle; + protected ObjectStoreDriver driver; + + @Override + public DataStoreLifeCycle getDataStoreLifeCycle() { + return lifeCycle; + } + + @Override + public String getName() { + return this.providerName; + } + + @Override + public boolean configure(Map params) { + lifeCycle = ComponentContext.inject(SimulatorObjectStoreLifeCycleImpl.class); + driver = ComponentContext.inject(SimulatorObjectStoreDriverImpl.class); + storeMgr.registerDriver(this.getName(), driver); + return true; + } + + @Override + public DataStoreDriver getDataStoreDriver() { + return this.driver; + } + + @Override + public HypervisorHostListener getHostListener() { + return null; + } + + @Override + public Set getTypes() { + Set types = new HashSet(); + types.add(DataStoreProviderType.OBJECT); + return types; + } +} diff --git a/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties new file mode 100644 index 00000000000..552f08c7a5c --- /dev/null +++ b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=storage-object-simulator +parent=storage diff --git a/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml new file mode 100644 index 00000000000..863b881d7f1 --- /dev/null +++ b/plugins/storage/object/simulator/src/main/resources/META-INF/cloudstack/storage-object-simulator/spring-storage-object-simulator-context.xml @@ -0,0 +1,31 @@ + + + + diff --git a/plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java b/plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java new file mode 100644 index 00000000000..57c7eee5cfb --- /dev/null +++ b/plugins/storage/object/simulator/src/test/java/org/apache/cloudstack/storage/datastore/provider/SimulatorObjectStoreProviderImplTest.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.datastore.provider; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; +import org.junit.Before; +import org.junit.Test; +import org.mockito.MockitoAnnotations; + +import java.util.Set; + +import static org.junit.Assert.assertEquals; + +public class SimulatorObjectStoreProviderImplTest { + + private SimulatorObjectStoreProviderImpl simulatorObjectStoreProviderImpl; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + simulatorObjectStoreProviderImpl = new SimulatorObjectStoreProviderImpl(); + } + + @Test + public void testGetName() { + String name = simulatorObjectStoreProviderImpl.getName(); + assertEquals("Simulator", name); + } + + @Test + public void testGetTypes() { + Set types = simulatorObjectStoreProviderImpl.getTypes(); + assertEquals(1, types.size()); + assertEquals("OBJECT", types.toArray()[0].toString()); + } +} diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 026c8f03407..a3532a79af4 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -294,6 +294,7 @@ import com.cloud.storage.Volume; import com.cloud.storage.Volume.Type; import com.cloud.storage.VolumeStats; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.GuestOSCategoryDao; import com.cloud.storage.dao.GuestOSDao; @@ -349,6 +350,10 @@ import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.apache.cloudstack.api.response.ObjectStoreResponse; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; + public class ApiDBUtils { private static ManagementServer s_ms; static AsyncJobManager s_asyncMgr; @@ -481,6 +486,9 @@ public class ApiDBUtils { static NicDao s_nicDao; static ResourceManagerUtil s_resourceManagerUtil; static SnapshotPolicyDetailsDao s_snapshotPolicyDetailsDao; + static ObjectStoreDao s_objectStoreDao; + + static BucketDao s_bucketDao; @Inject private ManagementServer ms; @@ -740,6 +748,11 @@ public class ApiDBUtils { @Inject SnapshotPolicyDetailsDao snapshotPolicyDetailsDao; + @Inject + private ObjectStoreDao objectStoreDao; + @Inject + private BucketDao bucketDao; + @PostConstruct void init() { s_ms = ms; @@ -871,6 +884,8 @@ public class ApiDBUtils { s_backupOfferingDao = backupOfferingDao; s_resourceIconDao = resourceIconDao; s_resourceManagerUtil = resourceManagerUtil; + s_objectStoreDao = objectStoreDao; + s_bucketDao = bucketDao; } // /////////////////////////////////////////////////////////// @@ -2208,4 +2223,12 @@ public class ApiDBUtils { public static NicSecondaryIpVO findSecondaryIpByIp4AddressAndNetworkId(String ip4Address, long networkId) { return s_nicSecondaryIpDao.findByIp4AddressAndNetworkId(ip4Address, networkId); } + + public static ObjectStoreResponse newObjectStoreResponse(ObjectStoreVO store) { + return s_objectStoreDao.newObjectStoreResponse(store); + } + + public static ObjectStoreResponse fillObjectStoreDetails(ObjectStoreResponse storeData, ObjectStoreVO store) { + return s_objectStoreDao.setObjectStoreResponse(storeData, store); + } } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 3649197a218..a6dc017074b 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -38,6 +38,8 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.BucketVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -63,6 +65,7 @@ import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.BackupOfferingResponse; import org.apache.cloudstack.api.response.BackupResponse; import org.apache.cloudstack.api.response.BackupScheduleResponse; +import org.apache.cloudstack.api.response.BucketResponse; import org.apache.cloudstack.api.response.CapabilityResponse; import org.apache.cloudstack.api.response.CapacityResponse; import org.apache.cloudstack.api.response.ClusterResponse; @@ -119,6 +122,7 @@ import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.NicExtraDhcpOptionResponse; import org.apache.cloudstack.api.response.NicResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.OvsProviderResponse; import org.apache.cloudstack.api.response.PhysicalNetworkResponse; import org.apache.cloudstack.api.response.PodResponse; @@ -196,10 +200,14 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.object.Bucket; +import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.usage.UsageService; import org.apache.cloudstack.usage.UsageTypes; @@ -263,7 +271,6 @@ import com.cloud.gpu.GPU; import com.cloud.host.ControlState; import com.cloud.host.Host; import com.cloud.host.HostVO; -import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.network.GuestVlan; import com.cloud.network.GuestVlanRange; @@ -470,6 +477,9 @@ public class ApiResponseHelper implements ResponseGenerator { @Inject UserDataDao userDataDao; + @Inject + ObjectStoreDao _objectStoreDao; + @Override public UserResponse createUserResponse(User user) { UserAccountJoinVO vUser = ApiDBUtils.newUserView(user); @@ -4309,6 +4319,10 @@ public class ApiResponseHelper implements ResponseGenerator { } usageRecResponse.setDescription(builder.toString()); } + } else if (usageRecord.getUsageType() == UsageTypes.BUCKET) { + BucketVO bucket = _entityMgr.findByIdIncludingRemoved(BucketVO.class, usageRecord.getUsageId().toString()); + usageRecResponse.setUsageId(bucket.getUuid()); + usageRecResponse.setResourceName(bucket.getName()); } if(resourceTagResponseMap != null && resourceTagResponseMap.get(resourceId + ":" + resourceType) != null) { usageRecResponse.setTags(resourceTagResponseMap.get(resourceId + ":" + resourceType)); @@ -5109,4 +5123,40 @@ public class ApiResponseHelper implements ResponseGenerator { return quarantinedIpsResponse; } + + public ObjectStoreResponse createObjectStoreResponse(ObjectStore os) { + ObjectStoreResponse objectStoreResponse = new ObjectStoreResponse(); + objectStoreResponse.setId(os.getUuid()); + objectStoreResponse.setName(os.getName()); + objectStoreResponse.setProviderName(os.getProviderName()); + objectStoreResponse.setObjectName("objectstore"); + return objectStoreResponse; + } + + @Override + public BucketResponse createBucketResponse(Bucket bucket) { + BucketResponse bucketResponse = new BucketResponse(); + bucketResponse.setName(bucket.getName()); + bucketResponse.setId(bucket.getUuid()); + bucketResponse.setCreated(bucket.getCreated()); + bucketResponse.setState(bucket.getState()); + bucketResponse.setSize(bucket.getSize()); + if(bucket.getQuota() != null) { + bucketResponse.setQuota(bucket.getQuota()); + } + bucketResponse.setVersioning(bucket.isVersioning()); + bucketResponse.setEncryption(bucket.isEncryption()); + bucketResponse.setObjectLock(bucket.isObjectLock()); + bucketResponse.setPolicy(bucket.getPolicy()); + bucketResponse.setBucketURL(bucket.getBucketURL()); + bucketResponse.setAccessKey(bucket.getAccessKey()); + bucketResponse.setSecretKey(bucket.getSecretKey()); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); + bucketResponse.setObjectStoragePoolId(objectStoreVO.getUuid()); + bucketResponse.setObjectStoragePool(objectStoreVO.getName()); + bucketResponse.setObjectName("bucket"); + bucketResponse.setProvider(objectStoreVO.getProviderName()); + populateAccount(bucketResponse, bucket.getAccountId()); + return bucketResponse; + } } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 98f162697d2..fd3e3298338 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -83,6 +83,7 @@ import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResult import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.snapshot.ListSnapshotsCmdByAdmin; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; @@ -126,6 +127,7 @@ import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -156,6 +158,8 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; @@ -261,6 +265,7 @@ import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.service.ServiceOfferingVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.service.dao.ServiceOfferingDetailsDao; +import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.ScopeType; @@ -275,6 +280,7 @@ import com.cloud.storage.VMTemplateVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; @@ -310,6 +316,8 @@ import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.DomainRouterDao; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; +import org.apache.cloudstack.api.response.BucketResponse; import static com.cloud.vm.VmDetailConstants.SSH_PUBLIC_KEY; @@ -543,6 +551,12 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private SnapshotJoinDao snapshotJoinDao; + @Inject + private ObjectStoreDao objectStoreDao; + + @Inject + private BucketDao bucketDao; + @Inject EntityManager entityManager; @@ -5030,6 +5044,158 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return new Pair<>(snapshots, count); } + public ListResponse searchForObjectStores(ListObjectStoragePoolsCmd cmd) { + Pair, Integer> result = searchForObjectStoresInternal(cmd); + ListResponse response = new ListResponse(); + + List poolResponses = ViewResponseHelper.createObjectStoreResponse(result.first().toArray(new ObjectStoreVO[result.first().size()])); + response.setResponses(poolResponses, result.second()); + return response; + } + + private Pair, Integer> searchForObjectStoresInternal(ListObjectStoragePoolsCmd cmd) { + + Object id = cmd.getId(); + Object name = cmd.getStoreName(); + String provider = cmd.getProvider(); + Object keyword = cmd.getKeyword(); + Long startIndex = cmd.getStartIndex(); + Long pageSize = cmd.getPageSizeVal(); + + Filter searchFilter = new Filter(ObjectStoreVO.class, "id", Boolean.TRUE, startIndex, pageSize); + + SearchBuilder sb = objectStoreDao.createSearchBuilder(); + sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct + // ids + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("provider", sb.entity().getProviderName(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + + if (keyword != null) { + SearchCriteria ssc = objectStoreDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + ssc.addOr("providerName", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (name != null) { + sc.setParameters("name", name); + } + + if (provider != null) { + sc.setParameters("provider", provider); + } + + // search Store details by ids + Pair, Integer> uniqueStorePair = objectStoreDao.searchAndCount(sc, searchFilter); + Integer count = uniqueStorePair.second(); + if (count.intValue() == 0) { + // empty result + return uniqueStorePair; + } + List uniqueStores = uniqueStorePair.first(); + Long[] osIds = new Long[uniqueStores.size()]; + int i = 0; + for (ObjectStoreVO v : uniqueStores) { + osIds[i++] = v.getId(); + } + List objectStores = objectStoreDao.searchByIds(osIds); + return new Pair<>(objectStores, count); + } + + + @Override + public ListResponse searchForBuckets(ListBucketsCmd listBucketsCmd) { + List buckets = searchForBucketsInternal(listBucketsCmd); + List bucketResponses = new ArrayList<>(); + for (BucketVO bucket : buckets) { + bucketResponses.add(responseGenerator.createBucketResponse(bucket)); + } + ListResponse response = new ListResponse<>(); + response.setResponses(bucketResponses, bucketResponses.size()); + return response; + } + + private List searchForBucketsInternal(ListBucketsCmd cmd) { + + Long id = cmd.getId(); + String name = cmd.getBucketName(); + Long storeId = cmd.getObjectStorageId(); + String keyword = cmd.getKeyword(); + Long startIndex = cmd.getStartIndex(); + Long pageSize = cmd.getPageSizeVal(); + Account caller = CallContext.current().getCallingAccount(); + List permittedAccounts = new ArrayList(); + + // Verify parameters + if (id != null) { + BucketVO bucket = bucketDao.findById(id); + if (bucket != null) { + accountMgr.checkAccess(CallContext.current().getCallingAccount(), null, true, bucket); + } + } + + List ids = getIdsListFromCmd(cmd.getId(), cmd.getIds()); + + Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), + cmd.isRecursive(), null); + accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccounts, domainIdRecursiveListProject, cmd.listAll(), false); + Long domainId = domainIdRecursiveListProject.first(); + Boolean isRecursive = domainIdRecursiveListProject.second(); + ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); + + Filter searchFilter = new Filter(BucketVO.class, "id", Boolean.TRUE, startIndex, pageSize); + + SearchBuilder sb = bucketDao.createSearchBuilder(); + accountMgr.buildACLSearchBuilder(sb, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + sb.select(null, Func.DISTINCT, sb.entity().getId()); // select distinct + // ids + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + + SearchCriteria sc = sb.create(); + accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); + + if (keyword != null) { + SearchCriteria ssc = bucketDao.createSearchCriteria(); + ssc.addOr("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + ssc.addOr("state", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.addAnd("name", SearchCriteria.Op.SC, ssc); + } + + if (id != null) { + sc.setParameters("id", id); + } + + if (name != null) { + sc.setParameters("name", name); + } + + setIdsListToSearchCriteria(sc, ids); + + // search Volume details by ids + Pair, Integer> uniqueBktPair = bucketDao.searchAndCount(sc, searchFilter); + Integer count = uniqueBktPair.second(); + if (count.intValue() == 0) { + // empty result + return uniqueBktPair.first(); + } + List uniqueBkts = uniqueBktPair.first(); + Long[] bktIds = new Long[uniqueBkts.size()]; + int i = 0; + for (BucketVO b : uniqueBkts) { + bktIds[i++] = b.getId(); + } + + return bucketDao.searchByIds(bktIds); + } + @Override public String getConfigComponentName() { return QueryService.class.getSimpleName(); diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index b909d69e80e..44096a799b7 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.Hashtable; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,6 +44,7 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.api.response.ProjectAccountResponse; import org.apache.cloudstack.api.response.ProjectInvitationResponse; import org.apache.cloudstack.api.response.ProjectResponse; @@ -58,6 +60,7 @@ import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.log4j.Logger; import com.cloud.api.ApiDBUtils; @@ -659,4 +662,20 @@ public class ViewResponseHelper { return new ArrayList(vrDataList.values()); } + public static List createObjectStoreResponse(ObjectStoreVO[] stores) { + Hashtable storeList = new Hashtable(); + // Initialise the storeList with the input data + for (ObjectStoreVO store : stores) { + ObjectStoreResponse storeData = storeList.get(store.getId()); + if (storeData == null) { + // first time encountering this store + storeData = ApiDBUtils.newObjectStoreResponse(store); + } else { + // update tags + storeData = ApiDBUtils.fillObjectStoreDetails(storeData, store); + } + storeList.put(store.getId(), storeData); + } + return new ArrayList<>(storeList.values()); + } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 7548b768327..37cdbc8c033 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -209,14 +209,17 @@ import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; import org.apache.cloudstack.api.command.admin.snapshot.ListSnapshotsCmdByAdmin; import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.AddImageStoreS3CMD; +import org.apache.cloudstack.api.command.admin.storage.AddObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.FindStoragePoolsForMigrationCmd; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd; @@ -227,6 +230,7 @@ import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForM import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateCloudToUseObjectStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStorageCapabilitiesCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; @@ -360,6 +364,10 @@ import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScalePolicyCmd import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScaleVmGroupCmd; import org.apache.cloudstack.api.command.user.autoscale.UpdateAutoScaleVmProfileCmd; import org.apache.cloudstack.api.command.user.autoscale.UpdateConditionCmd; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.DeleteBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; import org.apache.cloudstack.api.command.user.config.ListCapabilitiesCmd; import org.apache.cloudstack.api.command.user.consoleproxy.CreateConsoleEndpointCmd; import org.apache.cloudstack.api.command.user.event.ArchiveEventsCmd; @@ -3908,6 +3916,16 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(DeleteUserDataCmd.class); cmdList.add(ListUserDataCmd.class); cmdList.add(LinkUserDataToTemplateCmd.class); + + //object store APIs + cmdList.add(AddObjectStoragePoolCmd.class); + cmdList.add(ListObjectStoragePoolsCmd.class); + cmdList.add(UpdateObjectStoragePoolCmd.class); + cmdList.add(DeleteObjectStoragePoolCmd.class); + cmdList.add(CreateBucketCmd.class); + cmdList.add(UpdateBucketCmd.class); + cmdList.add(DeleteBucketCmd.class); + cmdList.add(ListBucketsCmd.class); return cmdList; } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 66063bee0f7..adbb43c03f5 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -54,9 +54,11 @@ import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaint import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; @@ -104,6 +106,9 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; @@ -115,6 +120,8 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.storage.object.ObjectStoreEntity; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.time.DateUtils; @@ -192,6 +199,7 @@ import com.cloud.service.dao.ServiceOfferingDetailsDao; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.Volume.Type; +import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.SnapshotDao; import com.cloud.storage.dao.StoragePoolHostDao; @@ -353,6 +361,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject protected UserVmManager userVmManager; + @Inject + protected ObjectStoreDao _objectStoreDao; + + @Inject + protected ObjectStoreDetailsDao _objectStoreDetailsDao; + + @Inject + protected BucketDao _bucketDao; protected List _discoverers; public List getDiscoverers() { @@ -3633,4 +3649,124 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C volumeTO.setIopsWriteRate(getDiskIopsWriteRate(offering, diskOffering)); } + @Override + @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_CREATE, eventDescription = "creating object storage") + public ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) + throws IllegalArgumentException, InvalidParameterValueException { + DataStoreProvider storeProvider = _dataStoreProviderMgr.getDataStoreProvider(providerName); + + if (storeProvider == null) { + throw new InvalidParameterValueException("can't find object store provider: " + providerName); + } + + // Check Unique object store name + ObjectStoreVO objectStore = _objectStoreDao.findByName(name); + if (objectStore != null) { + throw new InvalidParameterValueException("The object store with name " + name + " already exists, try creating with another name"); + } + + try { + // Check URL + UriUtils.validateUrl(url); + } catch (final Exception e) { + throw new InvalidParameterValueException(url + " is not a valid URL"); + } + + // Check Unique object store url + ObjectStoreVO objectStoreUrl = _objectStoreDao.findByUrl(url); + if (objectStoreUrl != null) { + throw new InvalidParameterValueException("The object store with url " + url + " already exists"); + } + + + Map params = new HashMap<>(); + params.put("url", url); + params.put("name", name); + params.put("providerName", storeProvider.getName()); + params.put("role", DataStoreRole.Object); + params.put("details", details); + + DataStoreLifeCycle lifeCycle = storeProvider.getDataStoreLifeCycle(); + + DataStore store; + try { + store = lifeCycle.initialize(params); + } catch (Exception e) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Failed to add object store: " + e.getMessage(), e); + } + throw new CloudRuntimeException("Failed to add object store: " + e.getMessage(), e); + } + + return (ObjectStore)_dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Object); + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_DELETE, eventDescription = "deleting object storage") + public boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd) { + final long storeId = cmd.getId(); + // Verify that object store exists + ObjectStoreVO store = _objectStoreDao.findById(storeId); + if (store == null) { + throw new InvalidParameterValueException("Object store with id " + storeId + " doesn't exist"); + } + + // Verify that there are no buckets in the store + List buckets = _bucketDao.listByObjectStoreId(storeId); + if(buckets != null && buckets.size() > 0) { + throw new InvalidParameterValueException("Cannot delete object store with buckets"); + } + + // ready to delete + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + _objectStoreDetailsDao.deleteDetails(storeId); + _objectStoreDao.remove(storeId); + } + }); + s_logger.debug("Successfully deleted object store with Id: "+storeId); + return true; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_OBJECT_STORE_UPDATE, eventDescription = "update object storage") + public ObjectStore updateObjectStore(Long id, UpdateObjectStoragePoolCmd cmd) { + + // Input validation + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(id); + if (objectStoreVO == null) { + throw new IllegalArgumentException("Unable to find object store with ID: " + id); + } + + if(cmd.getUrl() != null ) { + String url = cmd.getUrl(); + try { + // Check URL + UriUtils.validateUrl(url); + } catch (final Exception e) { + throw new InvalidParameterValueException(url + " is not a valid URL"); + } + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + String oldUrl = objectStoreVO.getUrl(); + objectStoreVO.setUrl(url); + _objectStoreDao.update(id, objectStoreVO); + //Update URL and check access + try { + objectStore.listBuckets(); + } catch (Exception e) { + //Revert to old URL on failure + objectStoreVO.setUrl(oldUrl); + _objectStoreDao.update(id, objectStoreVO); + throw new IllegalArgumentException("Unable to access Object Storage with URL: " + cmd.getUrl()); + } + } + + if(cmd.getName() != null ) { + objectStoreVO.setName(cmd.getName()); + } + _objectStoreDao.update(id, objectStoreVO); + s_logger.debug("Successfully updated object store with Id: "+id); + return objectStoreVO; + } } diff --git a/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java b/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java index 59879791ba3..6a9d40cad18 100644 --- a/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/annotation/AnnotationManagerImpl.java @@ -197,6 +197,7 @@ public final class AnnotationManagerImpl extends ManagerBase implements Annotati s_typeMap.put(EntityType.SYSTEM_VM, ApiCommandResourceType.SystemVm); s_typeMap.put(EntityType.AUTOSCALE_VM_GROUP, ApiCommandResourceType.AutoScaleVmGroup); s_typeMap.put(EntityType.MANAGEMENT_SERVER, ApiCommandResourceType.Host); + s_typeMap.put(EntityType.OBJECT_STORAGE, ApiCommandResourceType.ObjectStore); } public List getKubernetesClusterHelpers() { diff --git a/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java new file mode 100644 index 00000000000..bfd29cc0442 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/object/BucketApiServiceImpl.java @@ -0,0 +1,304 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.object; + +import com.amazonaws.services.s3.internal.BucketNameUtils; +import com.amazonaws.services.s3.model.IllegalBucketNameException; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.BucketVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.dao.BucketDao; +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.usage.dao.BucketStatisticsDao; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.user.bucket.CreateBucketCmd; +import org.apache.cloudstack.api.command.user.bucket.UpdateBucketCmd; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class BucketApiServiceImpl extends ManagerBase implements BucketApiService, Configurable { + private final static Logger s_logger = Logger.getLogger(BucketApiServiceImpl.class); + + @Inject + private ObjectStoreDao _objectStoreDao; + @Inject + DataStoreManager _dataStoreMgr; + @Inject + private BucketDao _bucketDao; + @Inject + private AccountManager _accountMgr; + + @Inject + private BucketStatisticsDao _bucketStatisticsDao; + + private ScheduledExecutorService _executor = null; + + private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; + + protected BucketApiServiceImpl() { + + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + _executor = Executors.newScheduledThreadPool(1, new NamedThreadFactory("Bucket-Usage")); + return true; + } + + @Override + public boolean start() { + _executor.scheduleWithFixedDelay(new BucketUsageTask(), 60L, 3600L, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + _executor.shutdown(); + return true; + } + + @Override + public String getConfigComponentName() { + return BucketApiService.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + }; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", create = true) + public Bucket allocBucket(CreateBucketCmd cmd) { + try { + BucketNameUtils.validateBucketName(cmd.getBucketName()); + } catch (IllegalBucketNameException e) { + s_logger.error("Invalid Bucket Name: " +cmd.getBucketName(), e); + throw new InvalidParameterValueException("Invalid Bucket Name: "+e.getMessage()); + } + //ToDo check bucket exists + long ownerId = cmd.getEntityOwnerId(); + Account owner = _accountMgr.getActiveAccountById(ownerId); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(cmd.getObjectStoragePoolId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + try { + if(!objectStore.createUser(ownerId)) { + s_logger.error("Failed to create user in objectstore "+ objectStore.getName()); + return null; + } + } catch (CloudRuntimeException e) { + s_logger.error("Error while checking object store user.", e); + return null; + } + + BucketVO bucket = new BucketVO(ownerId, owner.getDomainId(), cmd.getObjectStoragePoolId(), cmd.getBucketName(), cmd.getQuota(), + cmd.isVersioning(), cmd.isEncryption(), cmd.isObjectLocking(), cmd.getPolicy()); + _bucketDao.persist(bucket); + return bucket; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_CREATE, eventDescription = "creating bucket", async = true) + public Bucket createBucket(CreateBucketCmd cmd) { + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(cmd.getObjectStoragePoolId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + BucketVO bucket = _bucketDao.findById(cmd.getEntityId()); + boolean objectLock = false; + boolean bucketCreated = false; + if(cmd.isObjectLocking()) { + objectLock = true; + } + try { + objectStore.createBucket(bucket, objectLock); + bucketCreated = true; + + if (cmd.isVersioning()) { + objectStore.setBucketVersioning(bucket.getName()); + } + + if (cmd.isEncryption()) { + objectStore.setBucketEncryption(bucket.getName()); + } + + if (cmd.getQuota() != null) { + objectStore.setQuota(bucket.getName(), cmd.getQuota()); + } + + if (cmd.getPolicy() != null) { + objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + } + + bucket.setState(Bucket.State.Created); + _bucketDao.update(bucket.getId(), bucket); + } catch (Exception e) { + s_logger.debug("Failed to create bucket with name: "+bucket.getName(), e); + if(bucketCreated) { + objectStore.deleteBucket(bucket.getName()); + } + _bucketDao.remove(bucket.getId()); + throw new CloudRuntimeException("Failed to create bucket with name: "+bucket.getName()+". "+e.getMessage()); + } + return bucket; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_DELETE, eventDescription = "deleting bucket") + public boolean deleteBucket(long bucketId, Account caller) { + Bucket bucket = _bucketDao.findById(bucketId); + if (bucket == null) { + throw new InvalidParameterValueException("Unable to find bucket with ID: " + bucketId); + } + _accountMgr.checkAccess(caller, null, true, bucket); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + if (objectStore.deleteBucket(bucket.getName())) { + return _bucketDao.remove(bucketId); + } + return false; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_BUCKET_UPDATE, eventDescription = "updating bucket") + public boolean updateBucket(UpdateBucketCmd cmd, Account caller) { + BucketVO bucket = _bucketDao.findById(cmd.getId()); + if (bucket == null) { + throw new InvalidParameterValueException("Unable to find bucket with ID: " + cmd.getId()); + } + _accountMgr.checkAccess(caller, null, true, bucket); + ObjectStoreVO objectStoreVO = _objectStoreDao.findById(bucket.getObjectStoreId()); + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + try { + if (cmd.getEncryption() != null) { + if (cmd.getEncryption()) { + objectStore.setBucketEncryption(bucket.getName()); + } else { + objectStore.deleteBucketEncryption(bucket.getName()); + } + bucket.setEncryption(cmd.getEncryption()); + } + + if (cmd.getVersioning() != null) { + if (cmd.getVersioning()) { + objectStore.setBucketVersioning(bucket.getName()); + } else { + objectStore.deleteBucketVersioning(bucket.getName()); + } + bucket.setVersioning(cmd.getVersioning()); + } + + if (cmd.getPolicy() != null) { + objectStore.setBucketPolicy(bucket.getName(), cmd.getPolicy()); + bucket.setPolicy(cmd.getPolicy()); + } + + if (cmd.getQuota() != null) { + objectStore.setQuota(bucket.getName(), cmd.getQuota()); + bucket.setQuota(cmd.getQuota()); + } + _bucketDao.update(bucket.getId(), bucket); + } catch (Exception e) { + throw new CloudRuntimeException("Error while updating bucket: " +bucket.getName() +". "+e.getMessage()); + } + + return true; + } + + public void getBucketUsage() { + //ToDo track usage one last time when object store or bucket is removed + List objectStores = _objectStoreDao.listObjectStores(); + for(ObjectStoreVO objectStoreVO: objectStores) { + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Map bucketSizes = objectStore.getAllBucketsUsage(); + List buckets = _bucketDao.listByObjectStoreId(objectStoreVO.getId()); + for(BucketVO bucket : buckets) { + Long size = bucketSizes.get(bucket.getName()); + if( size != null){ + bucket.setSize(size); + _bucketDao.update(bucket.getId(), bucket); + } + } + } + } + + private class BucketUsageTask extends ManagedContextRunnable { + public BucketUsageTask() { + } + + @Override + protected void runInContext() { + GlobalLock scanLock = GlobalLock.getInternLock("BucketUsage"); + try { + if (scanLock.lock(ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION)) { + try { + List objectStores = _objectStoreDao.listObjectStores(); + for(ObjectStoreVO objectStoreVO: objectStores) { + ObjectStoreEntity objectStore = (ObjectStoreEntity)_dataStoreMgr.getDataStore(objectStoreVO.getId(), DataStoreRole.Object); + Map bucketSizes = objectStore.getAllBucketsUsage(); + List buckets = _bucketDao.listByObjectStoreId(objectStoreVO.getId()); + for(BucketVO bucket : buckets) { + Long size = bucketSizes.get(bucket.getName()); + if( size != null){ + bucket.setSize(size); + _bucketDao.update(bucket.getId(), bucket); + + //Update Bucket Usage stats + BucketStatisticsVO bucketStatisticsVO = _bucketStatisticsDao.findBy(bucket.getAccountId(), bucket.getId()); + if(bucketStatisticsVO != null) { + bucketStatisticsVO.setSize(size); + _bucketStatisticsDao.update(bucketStatisticsVO.getId(), bucketStatisticsVO); + } else { + bucketStatisticsVO = new BucketStatisticsVO(bucket.getAccountId(), bucket.getId()); + bucketStatisticsVO.setSize(size); + _bucketStatisticsDao.persist(bucketStatisticsVO); + } + } + } + } + s_logger.debug("Completed updating bucket usage for all object stores"); + } catch (Exception e) { + s_logger.error("Error while fetching bucket usage", e); + } finally { + scanLock.unlock(); + } + } + } finally { + scanLock.releaseRef(); + } + } + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index ca3a77b8e89..7227264e229 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -352,6 +352,7 @@ class="com.cloud.tags.ResourceManagerUtilImpl"/> + diff --git a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java index 764d9437fa9..4af84cfa814 100644 --- a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java +++ b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java @@ -27,6 +27,8 @@ import com.cloud.network.Network; import com.cloud.network.VNF; import com.cloud.network.dao.NetworkVO; import com.cloud.server.ResourceTag; +import com.cloud.storage.BucketVO; +import com.cloud.storage.dao.BucketDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.AccountVO; @@ -40,12 +42,17 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; +import org.apache.cloudstack.api.command.user.bucket.ListBucketsCmd; import org.apache.cloudstack.api.command.user.event.ListEventsCmd; import org.apache.cloudstack.api.command.user.resource.ListDetailOptionsCmd; import org.apache.cloudstack.api.response.DetailOptionsResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ObjectStoreResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; +import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -66,6 +73,8 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -95,6 +104,12 @@ public class QueryManagerImplTest { @Mock SearchCriteria searchCriteriaMock; + @Mock + ObjectStoreDao objectStoreDao; + + @Mock + BucketDao bucketDao; + private AccountVO account; private UserVO user; @@ -288,4 +303,41 @@ public class QueryManagerImplTest { Assert.assertTrue(set.contains(domainId)); } + + @Test + public void testSearchForObjectStores() { + ListObjectStoragePoolsCmd cmd = new ListObjectStoragePoolsCmd(); + List objectStores = new ArrayList<>(); + ObjectStoreVO os1 = new ObjectStoreVO(); + os1.setName("MinIOStore"); + ObjectStoreVO os2 = new ObjectStoreVO(); + os1.setName("Simulator"); + objectStores.add(os1); + objectStores.add(os2); + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + ObjectStoreVO objectStoreVO = Mockito.mock(ObjectStoreVO.class); + when(sb.entity()).thenReturn(objectStoreVO); + when(objectStoreDao.createSearchBuilder()).thenReturn(sb); + when(objectStoreDao.searchAndCount(any(), any())).thenReturn(new Pair<>(objectStores, 2)); + ListResponse result = queryManagerImplSpy.searchForObjectStores(cmd); + assertEquals(2, result.getCount().intValue()); + } + + @Test + public void testSearchForBuckets() { + ListBucketsCmd listBucketsCmd = new ListBucketsCmd(); + List buckets = new ArrayList<>(); + BucketVO b1 = new BucketVO(); + b1.setName("test-bucket-1"); + BucketVO b2 = new BucketVO(); + b2.setName("test-bucket-1"); + buckets.add(b1); + buckets.add(b2); + SearchBuilder sb = Mockito.mock(SearchBuilder.class); + BucketVO bucketVO = Mockito.mock(BucketVO.class); + when(sb.entity()).thenReturn(bucketVO); + when(bucketDao.createSearchBuilder()).thenReturn(sb); + when(bucketDao.searchAndCount(any(), any())).thenReturn(new Pair<>(buckets, 2)); + queryManagerImplSpy.searchForBuckets(listBucketsCmd); + } } diff --git a/test/integration/smoke/test_bucket.py b/test/integration/smoke/test_bucket.py new file mode 100644 index 00000000000..7d92ea98b07 --- /dev/null +++ b/test/integration/smoke/test_bucket.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" BVT tests for Bucket Operations""" + +#Import Local Modules +from marvin.cloudstackTestCase import * +from nose.plugins.attrib import attr +from marvin.lib.base import (ObjectStoragePool, Bucket) +from marvin.lib.utils import (cleanup_resources) + +_multiprocess_shared_ = True + +class TestObjectStore(cloudstackTestCase): + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + #Clean up, terminate the created resources + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["smoke"], required_hardware="false") + def test_01_create_bucket(self): + """Test to create bucket in object store + + """ + + object_store = ObjectStoragePool.create( + self.apiclient, + "testOS-9", + "http://192.168.0.1", + "Simulator", + None + ) + + self.debug("Created Object Store with ID: %s" % object_store.id) + + bucket = Bucket.create( + self.apiclient, + "mybucket", + object_store.id + ) + + list_buckets_response = Bucket.list( + self.apiclient, + id=bucket.id + ) + + self.assertNotEqual( + len(list_buckets_response), + 0, + "Check List Bucket response" + ) + + bucket_response = list_buckets_response[0] + self.assertEqual( + object_store.id, + bucket_response.objectstorageid, + "Check object store id of the created Bucket" + ) + self.assertEqual( + "mybucket", + bucket_response.name, + "Check Name of the created Bucket" + ) + + bucket.update( + self.apiclient, + quota=100 + ) + + list_buckets_response_updated = Bucket.list( + self.apiclient, + id=bucket.id + ) + + bucket_response_updated = list_buckets_response_updated[0] + + self.assertEqual( + 100, + bucket_response_updated.quota, + "Check quota of the updated bucket" + ) + + self.cleanup.append(bucket) + self.cleanup.append(object_store) + + return diff --git a/test/integration/smoke/test_object_stores.py b/test/integration/smoke/test_object_stores.py new file mode 100644 index 00000000000..710fce646e1 --- /dev/null +++ b/test/integration/smoke/test_object_stores.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" BVT tests for Object Storage Pool""" + +#Import Local Modules +from marvin.cloudstackTestCase import cloudstackTestCase +from nose.plugins.attrib import attr +from marvin.lib.base import (ObjectStoragePool) +from marvin.lib.utils import (cleanup_resources) + +_multiprocess_shared_ = True + +class TestObjectStore(cloudstackTestCase): + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + return + + def tearDown(self): + try: + #Clean up, terminate the created resources + cleanup_resources(self.apiclient, self.cleanup) + + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags=["smoke"], required_hardware="false") + def test_01_create_object_store(self): + """Test to create object store + """ + + object_store = ObjectStoragePool.create( + self.apiclient, + "testOS-10", + "http://192.168.0.1", + "Simulator", + None + ) + + self.debug("Created Object Store with ID: %s" % object_store.id) + + list_object_stores_response = ObjectStoragePool.list( + self.apiclient, + id=object_store.id + ) + + self.assertNotEqual( + len(list_object_stores_response), + 0, + "Check List Object Store response" + ) + + object_store_response = list_object_stores_response[0] + self.assertEqual( + "Simulator", + object_store_response.providername, + "Check Provider of the created Object Store" + ) + self.assertEqual( + "testOS-10", + object_store_response.name, + "Check Name of the created Object Store" + ) + self.assertEqual( + "http://192.168.0.1", + object_store_response.url, + "Check URL of the created Object Store" + ) + + object_store.update( + self.apiclient, + name="updated_name" + ) + + list_object_stores_response_updated = ObjectStoragePool.list( + self.apiclient, + id=object_store.id + ) + + object_store_response_updated = list_object_stores_response_updated[0] + + self.assertEqual( + "updated_name", + object_store_response_updated.name, + "Check Name of the updated Object Store name" + ) + + self.cleanup.append(object_store) + + return diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index c07d27ae06c..8490dffa51d 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -259,7 +259,15 @@ known_categories = { 'listQuarantinedIp': 'IP Quarantine', 'updateQuarantinedIp': 'IP Quarantine', 'removeQuarantinedIp': 'IP Quarantine', - 'Shutdown': 'Shutdown' + 'Shutdown': 'Shutdown', + 'addObjectStoragePool': 'Object Store', + 'listObjectStoragePools': 'Object Store', + 'deleteObjectStoragePool': 'Object Store', + 'updateObjectStoragePool': 'Object Store', + 'createBucket': 'Object Store', + 'updateBucket': 'Object Store', + 'deleteBucket': 'Object Store', + 'listBuckets': 'Object Store' } diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index a9827dcfaaf..39af1d4303b 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -7072,3 +7072,82 @@ class VnfAppliance: cmd.expunge = expunge [setattr(cmd, k, v) for k, v in list(kwargs.items())] apiclient.destroyVirtualMachine(cmd) + +class ObjectStoragePool: + + def __init__(self, items): + self.__dict__.update(items) + + """Manage Object Stores""" + @classmethod + def create(cls, apiclient, name, url, provider, services=None): + """Add Object Store""" + cmd = addObjectStoragePool.addObjectStoragePoolCmd() + cmd.name = name + cmd.url = url + cmd.provider = provider + if services: + if "details" in services: + cmd.details = services["details"] + + return ObjectStoragePool(apiclient.addObjectStoragePool(cmd).__dict__) + + def delete(self, apiclient): + """Delete Object Store""" + cmd = deleteObjectStoragePool.deleteObjectStoragePoolCmd() + cmd.id = self.id + apiclient.deleteObjectStoragePool(cmd) + + @classmethod + def list(cls, apiclient, **kwargs): + cmd = listObjectStoragePools.listObjectStoragePoolsCmd() + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + if 'account' in list(kwargs.keys()) and 'domainid' in list(kwargs.keys()): + cmd.listall = True + return (apiclient.listObjectStoragePools(cmd)) + + def update(self, apiclient, **kwargs): + """Update the Object Store""" + + cmd = updateObjectStoragePool.updateObjectStoragePoolCmd() + cmd.id = self.id + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + return apiclient.updateObjectStoragePool(cmd) + +class Bucket: + """Manage Bucket Life cycle""" + + def __init__(self, items): + self.__dict__.update(items) + + @classmethod + def create(cls, apiclient, name, objectstorageid, **kwargs): + """Create Bucket""" + cmd = createBucket.createBucketCmd() + cmd.name = name + cmd.objectstorageid = objectstorageid + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + + return Bucket(apiclient.createBucket(cmd).__dict__) + + def delete(self, apiclient): + """Delete Bucket""" + cmd = deleteBucket.deleteBucketCmd() + cmd.id = self.id + apiclient.deleteBucket(cmd) + + @classmethod + def list(cls, apiclient, **kwargs): + cmd = listBuckets.listBucketsCmd() + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + if 'account' in list(kwargs.keys()) and 'domainid' in list(kwargs.keys()): + cmd.listall = True + return (apiclient.listBuckets(cmd)) + + def update(self, apiclient, **kwargs): + """Update Bucket""" + + cmd = updateBucket.updateBucketCmd() + cmd.id = self.id + [setattr(cmd, k, v) for k, v in list(kwargs.items())] + return apiclient.updateBucket(cmd) diff --git a/ui/package-lock.json b/ui/package-lock.json index 0cbf08bd4a4..d0e0744f3ca 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -113,8 +113,7 @@ "version": "0.5.4", "resolved": "https://registry.npmjs.org/@apollographql/apollo-tools/-/apollo-tools-0.5.4.tgz", "integrity": "sha512-shM3q7rUbNyXVVRkQJQseXv6bnYM3BUma/eZhwXR4xsuM+bqWnJKvW7SAfRjP7LuSCocrexa5AXhjjawNHrIlw==", - "dev": true, - "requires": {} + "dev": true }, "@apollographql/graphql-playground-html": { "version": "1.6.27", @@ -1534,8 +1533,7 @@ "@fortawesome/vue-fontawesome": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@fortawesome/vue-fontawesome/-/vue-fontawesome-3.0.3.tgz", - "integrity": "sha512-KCPHi9QemVXGMrfuwf3nNnNo129resAIQWut9QTAMXmXqL2ErABC6ohd2yY5Ipq0CLWNbKHk8TMdTXL/Zf3ZhA==", - "requires": {} + "integrity": "sha512-KCPHi9QemVXGMrfuwf3nNnNo129resAIQWut9QTAMXmXqL2ErABC6ohd2yY5Ipq0CLWNbKHk8TMdTXL/Zf3ZhA==" }, "@gar/promisify": { "version": "1.1.3", @@ -3349,8 +3347,7 @@ "version": "4.5.19", "resolved": "https://registry.npmjs.org/@vue/cli-plugin-vuex/-/cli-plugin-vuex-4.5.19.tgz", "integrity": "sha512-DUmfdkG3pCdkP7Iznd87RfE9Qm42mgp2hcrNcYQYSru1W1gX2dG/JcW8bxmeGSa06lsxi9LEIc/QD1yPajSCZw==", - "dev": true, - "requires": {} + "dev": true }, "@vue/cli-service": { "version": "4.5.19", @@ -3861,8 +3858,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/@vue/preload-webpack-plugin/-/preload-webpack-plugin-1.1.2.tgz", "integrity": "sha512-LIZMuJk38pk9U9Ur4YzHjlIyMuxPlACdBIHH9/nGYVTsaGKOSnSuELiE8vS9wa+dJpIYspYUOqk+L1Q4pgHQHQ==", - "dev": true, - "requires": {} + "dev": true }, "@vue/reactivity": { "version": "3.2.37", @@ -3921,8 +3917,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/@vue/test-utils/-/test-utils-2.0.0.tgz", "integrity": "sha512-zL5kygNq7hONrO1CzaUGprEAklAX+pH8J1MPMCU3Rd2xtSYkZ+PmKU3oEDRg8VAGdL5lNJHzDgrud5amFPtirw==", - "dev": true, - "requires": {} + "dev": true }, "@vue/web-component-wrapper": { "version": "1.3.0", @@ -3934,6 +3929,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", "integrity": "sha512-C6wW5L+b7ogSDVqymbkkvuW9kruN//YisMED04xzeBBqjHa2FYnmvOlS6Xj68xWQRgWvI9cIglsjFowH/RJyEA==", + "dev": true, "requires": { "@webassemblyjs/helper-module-context": "1.9.0", "@webassemblyjs/helper-wasm-bytecode": "1.9.0", @@ -3943,22 +3939,26 @@ "@webassemblyjs/floating-point-hex-parser": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.9.0.tgz", - "integrity": "sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==" + "integrity": "sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==", + "dev": true }, "@webassemblyjs/helper-api-error": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.9.0.tgz", - "integrity": "sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==" + "integrity": "sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==", + "dev": true }, "@webassemblyjs/helper-buffer": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.9.0.tgz", - "integrity": "sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==" + "integrity": "sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==", + "dev": true }, "@webassemblyjs/helper-code-frame": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.9.0.tgz", "integrity": "sha512-ERCYdJBkD9Vu4vtjUYe8LZruWuNIToYq/ME22igL+2vj2dQ2OOujIZr3MEFvfEaqKoVqpsFKAGsRdBSBjrIvZA==", + "dev": true, "requires": { "@webassemblyjs/wast-printer": "1.9.0" } @@ -3966,12 +3966,14 @@ "@webassemblyjs/helper-fsm": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.9.0.tgz", - "integrity": "sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==" + "integrity": "sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==", + "dev": true }, "@webassemblyjs/helper-module-context": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.9.0.tgz", "integrity": "sha512-MJCW8iGC08tMk2enck1aPW+BE5Cw8/7ph/VGZxwyvGbJwjktKkDK7vy7gAmMDx88D7mhDTCNKAW5tED+gZ0W8g==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0" } @@ -3979,12 +3981,14 @@ "@webassemblyjs/helper-wasm-bytecode": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.9.0.tgz", - "integrity": "sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==" + "integrity": "sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==", + "dev": true }, "@webassemblyjs/helper-wasm-section": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.9.0.tgz", "integrity": "sha512-XnMB8l3ek4tvrKUUku+IVaXNHz2YsJyOOmz+MMkZvh8h1uSJpSen6vYnw3IoQ7WwEuAhL8Efjms1ZWjqh2agvw==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-buffer": "1.9.0", @@ -3996,6 +4000,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.9.0.tgz", "integrity": "sha512-dcX8JuYU/gvymzIHc9DgxTzUUTLexWwt8uCTWP3otys596io0L5aW02Gb1RjYpx2+0Jus1h4ZFqjla7umFniTg==", + "dev": true, "requires": { "@xtuc/ieee754": "^1.2.0" } @@ -4004,6 +4009,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.9.0.tgz", "integrity": "sha512-ENVzM5VwV1ojs9jam6vPys97B/S65YQtv/aanqnU7D8aSoHFX8GyhGg0CMfyKNIHBuAVjy3tlzd5QMMINa7wpw==", + "dev": true, "requires": { "@xtuc/long": "4.2.2" } @@ -4011,12 +4017,14 @@ "@webassemblyjs/utf8": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.9.0.tgz", - "integrity": "sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==" + "integrity": "sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==", + "dev": true }, "@webassemblyjs/wasm-edit": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.9.0.tgz", "integrity": "sha512-FgHzBm80uwz5M8WKnMTn6j/sVbqilPdQXTWraSjBwFXSYGirpkSWE2R9Qvz9tNiTKQvoKILpCuTjBKzOIm0nxw==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-buffer": "1.9.0", @@ -4032,6 +4040,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.9.0.tgz", "integrity": "sha512-cPE3o44YzOOHvlsb4+E9qSqjc9Qf9Na1OO/BHFy4OI91XDE14MjFN4lTMezzaIWdPqHnsTodGGNP+iRSYfGkjA==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-wasm-bytecode": "1.9.0", @@ -4044,6 +4053,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.9.0.tgz", "integrity": "sha512-Qkjgm6Anhm+OMbIL0iokO7meajkzQD71ioelnfPEj6r4eOFuqm4YC3VBPqXjFyyNwowzbMD+hizmprP/Fwkl2A==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-buffer": "1.9.0", @@ -4055,6 +4065,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.9.0.tgz", "integrity": "sha512-9+wkMowR2AmdSWQzsPEjFU7njh8HTO5MqO8vjwEHuM+AMHioNqSBONRdr0NQQ3dVQrzp0s8lTcYqzUdb7YgELA==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-api-error": "1.9.0", @@ -4068,6 +4079,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.9.0.tgz", "integrity": "sha512-qsqSAP3QQ3LyZjNC/0jBJ/ToSxfYJ8kYyuiGvtn/8MK89VrNEfwj7BPQzJVHi0jGTRK2dGdJ5PRqhtjzoww+bw==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/floating-point-hex-parser": "1.9.0", @@ -4081,6 +4093,7 @@ "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.9.0.tgz", "integrity": "sha512-2J0nE95rHXHyQ24cWjMKJ1tqB/ds8z/cyeOZxJhcb+rW+SQASVjuznUSmdz5GpVJTzU8JkhYut0D3siFDD6wsA==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/wast-parser": "1.9.0", @@ -4099,12 +4112,20 @@ "@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", - "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==" + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true }, "@xtuc/long": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", - "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "dev": true + }, + "@zxing/text-encoding": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", + "integrity": "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA==", + "optional": true }, "abab": { "version": "2.0.6", @@ -4130,7 +4151,8 @@ "acorn": { "version": "6.4.2", "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", - "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==" + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", + "dev": true }, "acorn-globals": { "version": "4.3.4", @@ -4146,8 +4168,7 @@ "version": "5.3.2", "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} + "dev": true }, "acorn-walk": { "version": "6.2.0", @@ -4203,13 +4224,13 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz", "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==", - "requires": {} + "dev": true }, "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", - "requires": {} + "dev": true }, "alphanum-sort": { "version": "1.0.2", @@ -4377,14 +4398,14 @@ "resolved": "https://registry.npmjs.org/antd-theme-generator/-/antd-theme-generator-1.2.11.tgz", "integrity": "sha512-7A3lXyLb7eD7MXK7aSgZZ4DxQEdhZwyKhzIm70orUZPQJ8N8TWhZphyOWSGCe8yUqGQhi8PcpM2pLmTriZyKBw==", "requires": { - "glob": "*", - "hash.js": "*", - "less": "*", + "glob": "^7.1.3", + "hash.js": "^1.1.5", + "less": "^3.9.0", "less-bundle-promise": "^1.0.11", - "less-plugin-npm-import": "*", - "postcss": "*", + "less-plugin-npm-import": "^2.1.0", + "postcss": "^6.0.21", "postcss-less": "^3.1.4", - "strip-css-comments": "*" + "strip-css-comments": "^4.1.0" }, "dependencies": { "ansi-styles": { @@ -4985,7 +5006,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", "integrity": "sha512-P43ePfOAIupkguHUycrc4qJ9kz8ZiuOUijaETwX7THt0Y/GNK7v0aa8rY816xWjZ7rJdA5XdMcpVFTKMq+RvWg==", - "devOptional": true, + "dev": true, "requires": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" @@ -5107,8 +5128,7 @@ "version": "2.5.0", "resolved": "https://registry.npmjs.org/apollo-server-errors/-/apollo-server-errors-2.5.0.tgz", "integrity": "sha512-lO5oTjgiC3vlVg2RKr3RiXIIQ5pGXBFxYGGUkKDhTud3jMIhs+gel8L8zsEjKaKxkjHhCQAA/bcEfYiKkGQIvA==", - "dev": true, - "requires": {} + "dev": true }, "apollo-server-express": { "version": "2.25.4", @@ -5225,17 +5245,20 @@ "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", - "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==" + "integrity": "sha512-YVIQ82gZPGBebQV/a8dar4AitzCQs0jjXwMPZllpXMaGjXPYVUawSxQrRsjhjupyVxEvbHgUmIhKVlND+j02kA==", + "dev": true }, "arr-flatten": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==" + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true }, "arr-union": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", - "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==" + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "dev": true }, "array-equal": { "version": "1.0.0", @@ -5291,7 +5314,8 @@ "array-unique": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", - "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==" + "integrity": "sha512-SleRWjh9JUud2wH1hPs9rZBZ33H6T9HOiL0uwGnGx9FpE6wKGyfWugmbkEOIs6qWrZhg0LWeLziLrEwQJhs5mQ==", + "dev": true }, "array.prototype.flat": { "version": "1.3.0", @@ -5354,6 +5378,7 @@ "version": "1.5.0", "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.0.tgz", "integrity": "sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA==", + "dev": true, "requires": { "object-assign": "^4.1.1", "util": "0.10.3" @@ -5362,12 +5387,14 @@ "inherits": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", - "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==" + "integrity": "sha512-8nWq2nLTAwd02jTqJExUYFSD/fKq6VH9Y/oG2accc/kdI0V98Bag8d5a4gi3XHz73rDWa2PvTtvcWYquKqSENA==", + "dev": true }, "util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha512-5KiHfsmkqacuKjkRkdV7SsfDJ2EGiPsK92s2MhNSY0craxjTdKTtqKsJaCWp4LW33ZZ0OPUv1WO/TFvNQRiQxQ==", + "dev": true, "requires": { "inherits": "2.0.1" } @@ -5382,7 +5409,8 @@ "assign-symbols": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", - "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==" + "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", + "dev": true }, "ast-types": { "version": "0.13.3", @@ -5409,7 +5437,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", - "devOptional": true + "dev": true }, "async-limiter": { "version": "1.0.1", @@ -5447,7 +5475,8 @@ "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true }, "autoprefixer": { "version": "9.8.8", @@ -5464,6 +5493,11 @@ "postcss-value-parser": "^4.1.0" } }, + "available-typed-arrays": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", + "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" + }, "aws-sign2": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", @@ -5530,8 +5564,7 @@ "version": "7.0.0-bridge.0", "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-7.0.0-bridge.0.tgz", "integrity": "sha512-poPX9mZH/5CSanm50Q+1toVci6pv5KSRv/5TWCwtzQS5XEwn40BcCrgIeMFWP9CKKIniKXNxoIOnOq4VVlGXhg==", - "dev": true, - "requires": {} + "dev": true }, "babel-eslint": { "version": "10.1.0", @@ -6140,6 +6173,7 @@ "version": "0.11.2", "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, "requires": { "cache-base": "^1.0.1", "class-utils": "^0.3.5", @@ -6154,6 +6188,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, "requires": { "is-descriptor": "^1.0.0" } @@ -6162,6 +6197,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -6170,6 +6206,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -6178,6 +6215,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -6189,7 +6227,8 @@ "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true }, "batch": { "version": "0.6.1", @@ -6226,13 +6265,13 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", - "devOptional": true + "dev": true }, "bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "devOptional": true, + "dev": true, "requires": { "file-uri-to-path": "1.0.0" } @@ -6247,10 +6286,31 @@ "safe-buffer": "^5.1.1" } }, + "block-stream2": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", + "integrity": "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==", + "requires": { + "readable-stream": "^3.4.0" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "bluebird": { "version": "3.7.2", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", - "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true }, "bn.js": { "version": "5.2.1", @@ -6432,6 +6492,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, "requires": { "arr-flatten": "^1.1.0", "array-unique": "^0.3.2", @@ -6449,6 +6510,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -6460,6 +6522,11 @@ "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", "integrity": "sha512-cKV8tMCEpQs4hK/ik71d6LrPOnpkpGBR0wzxqr68g2m/LB2GxVYQroAjMJZRVM1Y4BCjCKc3vAamxSzOY2RP+w==" }, + "browser-or-node": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-1.3.0.tgz", + "integrity": "sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg==" + }, "browser-process-hrtime": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz", @@ -6563,6 +6630,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "dev": true, "requires": { "pako": "~1.0.5" } @@ -6626,8 +6694,7 @@ "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "dev": true + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" }, "buffer-fill": { "version": "1.0.0", @@ -6638,7 +6705,8 @@ "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true }, "buffer-indexof": { "version": "1.1.1", @@ -6660,7 +6728,8 @@ "builtin-status-codes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==" + "integrity": "sha512-HpGFw18DgFWlncDfjTa2rcQ4W88O1mC8e8yZ2AvQY5KDaktSTwo+KRf6nHK6FRI5FyRyb/5T6+TSxfP7QyGsmQ==", + "dev": true }, "builtins": { "version": "1.0.3", @@ -6711,6 +6780,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, "requires": { "collection-visit": "^1.0.0", "component-emitter": "^1.2.1", @@ -6821,7 +6891,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "get-intrinsic": "^1.0.2" @@ -6958,8 +7027,7 @@ "chartjs-adapter-moment": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/chartjs-adapter-moment/-/chartjs-adapter-moment-1.0.0.tgz", - "integrity": "sha512-PqlerEvQcc5hZLQ/NQWgBxgVQ4TRdvkW3c/t+SUEQSj78ia3hgLkf2VZ2yGJtltNbEEFyYGm+cA6XXevodYvWA==", - "requires": {} + "integrity": "sha512-PqlerEvQcc5hZLQ/NQWgBxgVQ4TRdvkW3c/t+SUEQSj78ia3hgLkf2VZ2yGJtltNbEEFyYGm+cA6XXevodYvWA==" }, "check-types": { "version": "8.0.3", @@ -6971,7 +7039,7 @@ "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "devOptional": true, + "dev": true, "requires": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -6987,7 +7055,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", - "devOptional": true, + "dev": true, "requires": { "fill-range": "^7.0.1" } @@ -6996,7 +7064,7 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", - "devOptional": true, + "dev": true, "requires": { "to-regex-range": "^5.0.1" } @@ -7005,7 +7073,7 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "devOptional": true, + "dev": true, "requires": { "is-glob": "^4.0.1" } @@ -7014,13 +7082,13 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "devOptional": true + "dev": true }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "devOptional": true, + "dev": true, "requires": { "is-number": "^7.0.0" } @@ -7035,7 +7103,8 @@ "chrome-trace-event": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz", - "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==" + "integrity": "sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==", + "dev": true }, "ci-info": { "version": "2.0.0", @@ -7060,6 +7129,7 @@ "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, "requires": { "arr-union": "^3.1.0", "define-property": "^0.2.5", @@ -7071,6 +7141,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -7364,6 +7435,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", "integrity": "sha512-lNkKvzEeMBBjUGHZ+q6z9pSJla0KWAQPvtzhEV9+iGyQYG+pBpl7xKDhxoNSOZH2hhv0v5k0y2yAM4o4SjoSkw==", + "dev": true, "requires": { "map-visit": "^1.0.0", "object-visit": "^1.0.0" @@ -7440,12 +7512,14 @@ "commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", - "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==" + "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==", + "dev": true }, "component-emitter": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", - "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==" + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true }, "compressible": { "version": "2.0.18", @@ -7508,6 +7582,7 @@ "version": "1.6.2", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "dev": true, "requires": { "buffer-from": "^1.0.0", "inherits": "^2.0.3", @@ -7578,7 +7653,8 @@ "console-browserify": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", - "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==", + "dev": true }, "console-control-strings": { "version": "1.1.0", @@ -7597,7 +7673,8 @@ "constants-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", - "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==" + "integrity": "sha512-xFxOwqIzR/e1k1gLiWEophSCMqXcwVHIH7akf7b/vxcUeGunlj3hvZaaqxwHsTgn+IndtkQJgSztIDWeumWJDQ==", + "dev": true }, "content-disposition": { "version": "0.5.4", @@ -7656,6 +7733,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "dev": true, "requires": { "aproba": "^1.1.1", "fs-write-stream-atomic": "^1.0.8", @@ -7669,6 +7747,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -7677,6 +7756,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -7686,7 +7766,8 @@ "copy-descriptor": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", - "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==" + "integrity": "sha512-XgZ0pFcakEUlbwQEVNg3+QAis1FyTL3Qel9FYy8pSkQqoG3PNoT0bOCQtOXcOkur21r2Eq2kI+IE+gsmAEVlYw==", + "dev": true }, "copy-to-clipboard": { "version": "3.3.1", @@ -8289,7 +8370,8 @@ "cyclist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", - "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==" + "integrity": "sha512-NJGVKPS81XejHcLhaLJS7plab0fK3slPh11mESeeDq2W4ZI5kUKK/LRRdVDvjJseojbPB7ZwjnyOybg3Igea/A==", + "dev": true }, "dashdash": { "version": "1.14.1", @@ -8375,7 +8457,8 @@ "decode-uri-component": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==" + "integrity": "sha512-hjf+xovcEn31w/EUYdTXQh/8smFL/dzYjohQGEIgjyNavaJfBY2p5F527Bo1VPATxv0VYTUC2bOcXvqFwk78Og==", + "dev": true }, "decompress": { "version": "4.2.1", @@ -8686,6 +8769,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, "requires": { "is-descriptor": "^1.0.2", "isobject": "^3.0.1" @@ -8695,6 +8779,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -8703,6 +8788,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -8711,6 +8797,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -8946,7 +9033,8 @@ "domain-browser": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", - "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==" + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "dev": true }, "domelementtype": { "version": "1.3.1", @@ -9186,6 +9274,7 @@ "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dev": true, "requires": { "end-of-stream": "^1.0.0", "inherits": "^2.0.1", @@ -9326,6 +9415,7 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.5.0.tgz", "integrity": "sha512-Nv9m36S/vxpsI+Hc4/ZGRs0n9mXqSWGGq49zxb/cJfPAQMbUtttJAlNPS4AQzaBdw/pKskw5bMbekT/Y7W/Wlg==", + "dev": true, "requires": { "graceful-fs": "^4.1.2", "memory-fs": "^0.5.0", @@ -9336,6 +9426,7 @@ "version": "0.5.0", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz", "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dev": true, "requires": { "errno": "^0.1.3", "readable-stream": "^2.0.1" @@ -9374,6 +9465,7 @@ "version": "0.1.8", "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz", "integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==", + "dev": true, "requires": { "prr": "~1.0.1" } @@ -9453,6 +9545,11 @@ "is-symbol": "^1.0.2" } }, + "es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==" + }, "escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -9669,8 +9766,7 @@ "version": "14.1.1", "resolved": "https://registry.npmjs.org/eslint-config-standard/-/eslint-config-standard-14.1.1.tgz", "integrity": "sha512-Z9B+VR+JIXRxz21udPTL9HpFMyoMUEeX1G251EQ6e05WD9aPVtVBn09XUmZ259wCMlCDmYDSZG62Hhm+ZTJcUg==", - "dev": true, - "requires": {} + "dev": true }, "eslint-import-resolver-node": { "version": "0.3.6", @@ -10036,8 +10132,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/eslint-plugin-standard/-/eslint-plugin-standard-4.1.0.tgz", "integrity": "sha512-ZL7+QRixjTR6/528YNGyDotyffm5OQst/sGxKDwGb9Uqs4In5Egi4+jbobhqJoyoCM6/7v/1A5fhQ7ScMtDjaQ==", - "dev": true, - "requires": {} + "dev": true }, "eslint-plugin-vue": { "version": "7.20.0", @@ -10072,6 +10167,7 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, "requires": { "esrecurse": "^4.1.0", "estraverse": "^4.1.1" @@ -10137,6 +10233,7 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, "requires": { "estraverse": "^5.2.0" }, @@ -10144,14 +10241,16 @@ "estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true } } }, "estraverse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==" + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true }, "estree-walker": { "version": "2.0.2", @@ -10185,7 +10284,8 @@ "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true }, "eventsource": { "version": "2.0.2", @@ -10236,6 +10336,7 @@ "version": "2.1.4", "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", "integrity": "sha512-w/ozOKR9Obk3qoWeY/WDi6MFta9AoMR+zud60mdnbniMcBxRuFJyDt2LdX/14A1UABeqk+Uk+LDfUpvoGKppZA==", + "dev": true, "requires": { "debug": "^2.3.3", "define-property": "^0.2.5", @@ -10250,6 +10351,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, "requires": { "ms": "2.0.0" } @@ -10258,6 +10360,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -10266,6 +10369,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -10273,7 +10377,8 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true } } }, @@ -10456,6 +10561,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", "integrity": "sha512-BwY5b5Ql4+qZoefgMj2NUmx+tehVTH/Kf4k1ZEtOHNFcm2wSxMRo992l6X3TIgni2eZVTZ85xMOjF31fwZAj6Q==", + "dev": true, "requires": { "assign-symbols": "^1.0.0", "is-extendable": "^1.0.1" @@ -10465,6 +10571,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, "requires": { "is-plain-object": "^2.0.4" } @@ -10473,6 +10580,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -10505,6 +10613,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, "requires": { "array-unique": "^0.3.2", "define-property": "^1.0.0", @@ -10520,6 +10629,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, "requires": { "is-descriptor": "^1.0.0" } @@ -10528,6 +10638,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -10536,6 +10647,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -10544,6 +10656,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -10552,6 +10665,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -10604,6 +10718,14 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-parser": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.3.0.tgz", + "integrity": "sha512-5Wln/SBrtlN37aboiNNFHfSALwLzpUx1vJhDgDVPKKG3JrNe8BWTUoNKqkeKk/HqNbKxC8nEAJaBydq30yHoLA==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -10705,7 +10827,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "devOptional": true + "dev": true }, "filename-reserved-regex": { "version": "2.0.0", @@ -10734,6 +10856,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", "integrity": "sha512-VcpLTWqWDiTerugjj8e3+esbg+skS3M9e54UuR3iCeIDMXCLTsAH8hTSzDQU/X6/6t3eYkOKoZSef2PlU6U1XQ==", + "dev": true, "requires": { "extend-shallow": "^2.0.1", "is-number": "^3.0.0", @@ -10745,12 +10868,18 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } } } }, + "filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==" + }, "finalhandler": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", @@ -10817,6 +10946,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, "requires": { "commondir": "^1.0.1", "make-dir": "^2.0.0", @@ -10827,6 +10957,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, "requires": { "pify": "^4.0.1", "semver": "^5.6.0" @@ -10835,12 +10966,14 @@ "pify": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", - "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true } } }, @@ -10911,6 +11044,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz", "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==", + "dev": true, "requires": { "inherits": "^2.0.3", "readable-stream": "^2.3.6" @@ -10925,7 +11059,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, "requires": { "is-callable": "^1.1.3" } @@ -10933,7 +11066,8 @@ "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", - "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==" + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "dev": true }, "forever-agent": { "version": "0.6.1", @@ -10960,6 +11094,7 @@ "version": "0.2.1", "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", "integrity": "sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==", + "dev": true, "requires": { "map-cache": "^0.2.2" } @@ -10974,6 +11109,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", + "dev": true, "requires": { "inherits": "^2.0.1", "readable-stream": "^2.0.0" @@ -11020,6 +11156,7 @@ "version": "1.0.10", "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", "integrity": "sha512-gehEzmPn2nAwr39eay+x3X34Ra+M2QlVUTLhkXPjWdeO8RF9kszk116avgBJM3ZyNHgHXBNx+VmPaFC36k0PzA==", + "dev": true, "requires": { "graceful-fs": "^4.1.2", "iferr": "^0.1.5", @@ -11036,6 +11173,7 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, "optional": true }, "fswin": { @@ -11047,8 +11185,7 @@ "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", - "dev": true + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, "function.prototype.name": { "version": "1.1.5", @@ -11120,7 +11257,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.2.tgz", "integrity": "sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==", - "dev": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -11168,7 +11304,8 @@ "get-value": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", - "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==" + "integrity": "sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==", + "dev": true }, "getpass": { "version": "0.1.7", @@ -11223,7 +11360,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", "integrity": "sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==", - "devOptional": true, + "dev": true, "requires": { "is-glob": "^3.1.0", "path-dirname": "^1.0.0" @@ -11233,7 +11370,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", "integrity": "sha512-UFpDDrPgM6qpnFNI+rh/p3bUaq9hKLZN8bMUWzxmcnZVS3omf4IPK+BrewlnWjO1WmUsMYuSjKh4UJuV4+Lqmw==", - "devOptional": true, + "dev": true, "requires": { "is-extglob": "^2.1.0" } @@ -11305,6 +11442,27 @@ "delegate": "^3.1.2" } }, + "gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "requires": { + "get-intrinsic": "^1.1.3" + }, + "dependencies": { + "get-intrinsic": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", + "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3" + } + } + } + }, "got": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", @@ -11391,8 +11549,7 @@ "version": "0.3.2", "resolved": "https://registry.npmjs.org/graphql-type-json/-/graphql-type-json-0.3.2.tgz", "integrity": "sha512-J+vjof74oMlCWXSvt0DOf2APEdZOCdubEvGDUAlqH//VBYcOYsGgRW7Xzorr44LvkjiuvecWc8fChxuZZbChtg==", - "dev": true, - "requires": {} + "dev": true }, "growly": { "version": "1.3.0", @@ -11442,7 +11599,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -11475,6 +11631,11 @@ "get-intrinsic": "^1.1.1" } }, + "has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" + }, "has-symbol-support-x": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", @@ -11484,8 +11645,7 @@ "has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" }, "has-to-string-tag-x": { "version": "1.4.1", @@ -11500,7 +11660,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dev": true, "requires": { "has-symbols": "^1.0.2" } @@ -11514,6 +11673,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", "integrity": "sha512-IBXk4GTsLYdQ7Rvt+GRBrFSVEkmuOUy4re0Xjd9kJSUQpnTrWR4/y9RpfexN9vkAPMFuQoeWKwqzPozRTlasGw==", + "dev": true, "requires": { "get-value": "^2.0.6", "has-values": "^1.0.0", @@ -11524,6 +11684,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", "integrity": "sha512-ODYZC64uqzmtfGMEAX/FvZiRyWLpAC3vYnNunURUnkGVTS+mI0smVsWaPydRBsE3g+ok7h960jChO8mFcWlHaQ==", + "dev": true, "requires": { "is-number": "^3.0.0", "kind-of": "^4.0.0" @@ -11533,6 +11694,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", "integrity": "sha512-24XsCxmEbRwEDbz/qz3stgin8TTzZ1ESR56OMCN0ujYg+vRutNSiOj9bHH9u85DKgXguraugV5sFuvbD4FW/hw==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -11945,7 +12107,8 @@ "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", - "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==" + "integrity": "sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg==", + "dev": true }, "https-proxy-agent": { "version": "5.0.1", @@ -11991,12 +12154,14 @@ "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true }, "iferr": { "version": "0.1.5", "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz", - "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==" + "integrity": "sha512-DUNFN5j7Tln0D+TxzloUjKB+CtVu6myn0JEFak6dG18mNt9YkQ6lzGCdafwofISZ1lLF3xRHJ98VKy9ynkcFaA==", + "dev": true }, "ignore": { "version": "4.0.6", @@ -12301,6 +12466,7 @@ "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", "integrity": "sha512-e1BM1qnDbMRG3ll2U9dSK0UMHuWOs3pY3AtcFsmvwPtKL3MML/Q86i+GilLfvqEs4GW+ExB91tQ3Ig9noDIZ+A==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -12309,6 +12475,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -12319,7 +12486,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dev": true, "requires": { "call-bind": "^1.0.2", "has-tostringtag": "^1.0.0" @@ -12344,7 +12510,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "devOptional": true, + "dev": true, "requires": { "binary-extensions": "^2.0.0" } @@ -12367,8 +12533,7 @@ "is-callable": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", - "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==", - "dev": true + "integrity": "sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w==" }, "is-ci": { "version": "2.0.0", @@ -12405,6 +12570,7 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", "integrity": "sha512-+w9D5ulSoBNlmw9OHn3U2v51SyoCd0he+bB3xMl62oijhrspxowjU+AIcDY0N3iEJbUEkB15IlMASQsxYigvXg==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -12413,6 +12579,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -12432,6 +12599,7 @@ "version": "0.1.6", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, "requires": { "is-accessor-descriptor": "^0.1.6", "is-data-descriptor": "^0.1.4", @@ -12441,7 +12609,8 @@ "kind-of": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==" + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true } } }, @@ -12460,13 +12629,14 @@ "is-extendable": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", - "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==" + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true }, "is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "devOptional": true + "dev": true }, "is-fullwidth-code-point": { "version": "1.0.0", @@ -12482,11 +12652,19 @@ "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", "dev": true }, + "is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "devOptional": true, + "dev": true, "requires": { "is-extglob": "^2.1.1" } @@ -12526,6 +12704,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", "integrity": "sha512-4cboCqIpliH+mAvFNegjZQ4kgKc3ZUhQVr3HvWbSh5q3WH2v82ct+T2Y1hdU5Gdtorx/cLifQjqCbL7bpznLTg==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -12534,6 +12713,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -12669,6 +12849,14 @@ "has-symbols": "^1.0.2" } }, + "is-typed-array": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", + "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", + "requires": { + "which-typed-array": "^1.1.11" + } + }, "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", @@ -12698,12 +12886,14 @@ "is-windows": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==" + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true }, "is-wsl": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz", - "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==" + "integrity": "sha512-gfygJYZ2gLTDlmbWMI0CE2MwnFzSN/2SZfkMlItC4K/JBlsWVDB0bO6XhqcY13YXE7iMcAJnzTCJjPiTeJJ0Mw==", + "dev": true }, "is-yarn-global": { "version": "0.3.0", @@ -12729,7 +12919,8 @@ "isobject": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==" + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true }, "isstream": { "version": "0.1.2", @@ -13794,8 +13985,7 @@ "version": "1.2.2", "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.2.tgz", "integrity": "sha512-olV41bKSMm8BdnuMsewT4jqlZ8+3TCARAXjZGT9jcoSnrfUnRCqnMoF9XEeoWjbzObpqF9dRhHQj0Xb9QdF6/w==", - "dev": true, - "requires": {} + "dev": true }, "jest-regex-util": { "version": "24.9.0", @@ -14840,7 +15030,8 @@ "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==" + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true }, "json-parse-even-better-errors": { "version": "2.3.1", @@ -14871,6 +15062,11 @@ "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, + "json-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-stream/-/json-stream-1.0.0.tgz", + "integrity": "sha512-H/ZGY0nIAg3QcOwE1QN/rK/Fa7gJn7Ii5obwp6zyPO4xiPNwpIMjqy2gwjBEGqzkF/vSWEIBQCBuN19hYiL6Qg==" + }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -14931,7 +15127,8 @@ "kind-of": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", - "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true }, "kleur": { "version": "3.0.3", @@ -15224,7 +15421,8 @@ "loader-runner": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", - "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==" + "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==", + "dev": true }, "loader-utils": { "version": "2.0.2", @@ -15491,12 +15689,14 @@ "map-cache": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", - "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==" + "integrity": "sha512-8y/eV9QQZCiyn1SprXSrCmqJN0yNRATe+PO8ztwqrvrbdRLA3eYJF0yaR0YayLWkMbsQSKWS9N2gPcGEc4UsZg==", + "dev": true }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", "integrity": "sha512-4y7uGv8bd2WdM9vpQsiQNo41Ln1NvhvDRuVt0k2JZQ+ezN2uaQes7lZeZ+QQUHOLQAtDaBJ+7wCbi+ab/KFs+w==", + "dev": true, "requires": { "object-visit": "^1.0.0" } @@ -15542,6 +15742,7 @@ "version": "0.4.1", "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", "integrity": "sha512-cda4JKCxReDXFXRqOHPQscuIYg1PvxbE2S2GP45rnwfEK+vZaXC8C1OFvdHIbgw0DLzowXGVoxLaAmlgRy14GQ==", + "dev": true, "requires": { "errno": "^0.1.3", "readable-stream": "^2.0.1" @@ -15590,6 +15791,7 @@ "version": "3.1.10", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, "requires": { "arr-diff": "^4.0.0", "array-unique": "^0.3.2", @@ -15758,6 +15960,80 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==" }, + "minio": { + "version": "7.0.33", + "resolved": "https://registry.npmjs.org/minio/-/minio-7.0.33.tgz", + "integrity": "sha512-8wXGH98nZiLPe2xZhMV7UJ+48L1UlhgekxgpUhJWMO1h24TvZ0wUjtIt9e7DfNACopXh1spL8iuDQD7Lrq8Upw==", + "requires": { + "async": "^3.1.0", + "block-stream2": "^2.0.0", + "browser-or-node": "^1.3.0", + "buffer-crc32": "^0.2.13", + "crypto-browserify": "^3.12.0", + "es6-error": "^4.1.1", + "fast-xml-parser": "^4.1.3", + "ipaddr.js": "^2.0.1", + "json-stream": "^1.0.0", + "lodash": "^4.17.21", + "mime-types": "^2.1.14", + "mkdirp": "^0.5.1", + "query-string": "^7.1.1", + "through2": "^3.0.1", + "web-encoding": "^1.1.5", + "xml": "^1.0.0", + "xml2js": "^0.4.15" + }, + "dependencies": { + "async": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", + "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==" + }, + "decode-uri-component": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==" + }, + "ipaddr.js": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", + "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==" + }, + "mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "requires": { + "minimist": "^1.2.6" + } + }, + "query-string": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", + "integrity": "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==", + "requires": { + "decode-uri-component": "^0.2.2", + "filter-obj": "^1.1.0", + "split-on-first": "^1.0.0", + "strict-uri-encode": "^2.0.0" + } + }, + "strict-uri-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==" + }, + "through2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", + "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "2 || 3" + } + } + } + }, "minipass": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.4.tgz", @@ -15831,6 +16107,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==", + "dev": true, "requires": { "concat-stream": "^1.5.0", "duplexify": "^3.4.2", @@ -15853,6 +16130,7 @@ "version": "1.3.2", "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, "requires": { "for-in": "^1.0.2", "is-extendable": "^1.0.1" @@ -15862,6 +16140,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, "requires": { "is-plain-object": "^2.0.4" } @@ -15870,6 +16149,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -15905,6 +16185,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", "integrity": "sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==", + "dev": true, "requires": { "aproba": "^1.1.1", "copy-concurrently": "^1.0.0", @@ -15918,6 +16199,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -15926,6 +16208,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -15979,6 +16262,7 @@ "version": "2.16.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.16.0.tgz", "integrity": "sha512-UdAqHyFngu7TfQKsCBgAA6pWDkT8MAO7d0jyOecVhN5354xbLqdn8mV9Tat9gepAupm0bt2DbeaSC8vS52MuFA==", + "dev": true, "optional": true }, "nanoid": { @@ -15990,6 +16274,7 @@ "version": "1.2.13", "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, "requires": { "arr-diff": "^4.0.0", "array-unique": "^0.3.2", @@ -16065,7 +16350,8 @@ "neo-async": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==" + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true }, "nested-error-stacks": { "version": "2.0.1", @@ -16195,6 +16481,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==", + "dev": true, "requires": { "assert": "^1.1.1", "browserify-zlib": "^0.2.0", @@ -16225,6 +16512,7 @@ "version": "4.9.2", "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dev": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4", @@ -16234,7 +16522,8 @@ "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==" + "integrity": "sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==", + "dev": true } } }, @@ -16324,7 +16613,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "devOptional": true + "dev": true }, "normalize-range": { "version": "0.1.2", @@ -16505,6 +16794,7 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", "integrity": "sha512-79LYn6VAb63zgtmAteVOWo9Vdj71ZVBy3Pbse+VqxDpEP83XuujMrGqHIwAXJ5I/aM0zU7dIyIAhifVTPrNItQ==", + "dev": true, "requires": { "copy-descriptor": "^0.1.0", "define-property": "^0.2.5", @@ -16515,6 +16805,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -16523,6 +16814,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -16567,6 +16859,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", "integrity": "sha512-GBaMwwAVK9qbQN3Scdo0OyvgPW7l3lnaVMj84uTOZlswkX0KpF6fyDBJhtTthf7pymztoN36/KEr1DyhF96zEA==", + "dev": true, "requires": { "isobject": "^3.0.0" } @@ -16599,6 +16892,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", "integrity": "sha512-tqa/UMy/CCoYmj+H5qc07qvSL9dqcs/WZENZ1JbtWBlATP+iVOe778gE6MSijnyCnORzDuX6hU+LA4SZ09YjFQ==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -16774,7 +17068,8 @@ "os-browserify": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==" + "integrity": "sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A==", + "dev": true }, "os-tmpdir": { "version": "1.0.2", @@ -16917,12 +17212,14 @@ "pako": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true }, "parallel-transform": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==", + "dev": true, "requires": { "cyclist": "^1.0.1", "inherits": "^2.0.3", @@ -17020,18 +17317,20 @@ "pascalcase": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", - "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==" + "integrity": "sha512-XHXfu/yOQRy9vYOtUDVMN60OEJjW013GoObG1o+xwQTpB9eYJX/BjXMsdW13ZDPruFhYYn0AG22w0xgQMwl3Nw==", + "dev": true }, "path-browserify": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz", - "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==" + "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==", + "dev": true }, "path-dirname": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", "integrity": "sha512-ALzNPpyNq9AqXMBjeymIjFDAkAFH06mHJH/cSBHAgU0s4vfpBn6b2nf8tiRLvagKD8RbTpq2FKTBg7cl9l3c7Q==", - "devOptional": true + "dev": true }, "path-exists": { "version": "4.0.0", @@ -17107,7 +17406,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "devOptional": true + "dev": true }, "pid-from-port": { "version": "1.1.3", @@ -17208,6 +17507,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, "requires": { "find-up": "^3.0.0" }, @@ -17216,6 +17516,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, "requires": { "locate-path": "^3.0.0" } @@ -17224,6 +17525,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, "requires": { "p-locate": "^3.0.0", "path-exists": "^3.0.0" @@ -17233,6 +17535,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, "requires": { "p-limit": "^2.0.0" } @@ -17240,7 +17543,8 @@ "path-exists": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==" + "integrity": "sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==", + "dev": true } } }, @@ -17293,7 +17597,8 @@ "posix-character-classes": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", - "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==" + "integrity": "sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg==", + "dev": true }, "postcss": { "version": "7.0.39", @@ -18001,7 +18306,8 @@ "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==" + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true }, "process-exists": { "version": "3.1.0", @@ -18064,7 +18370,8 @@ "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", - "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==" + "integrity": "sha512-yPw4Sng1gWghHQWj0B3ZggWUm4qVbPwPFcRG8KyxiU7J2OHFSoEHKS+EZ3fv5l1t9CyCiop6l/ZYeWbrgoQejw==", + "dev": true }, "ps-list": { "version": "4.1.0", @@ -18120,6 +18427,7 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "dev": true, "requires": { "duplexify": "^3.6.0", "inherits": "^2.0.3", @@ -18130,6 +18438,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dev": true, "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -18180,12 +18489,14 @@ "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "dev": true }, "querystring-es3": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", - "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==" + "integrity": "sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA==", + "dev": true }, "querystringify": { "version": "2.2.0", @@ -18804,7 +19115,7 @@ "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "devOptional": true, + "dev": true, "requires": { "picomatch": "^2.2.1" } @@ -18863,6 +19174,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, "requires": { "extend-shallow": "^3.0.2", "safe-regex": "^1.1.0" @@ -18948,7 +19260,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", "integrity": "sha512-/hS+Y0u3aOfIETiaiirUFwDBDzmXPvO+jAfKTitUngIPzdKc6Z0LoFjM/CK5PL4C+eKwHohlHAb6H0VFfmmUsw==", - "devOptional": true + "dev": true }, "renderkid": { "version": "2.0.7", @@ -19024,12 +19336,14 @@ "repeat-element": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.4.tgz", - "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==" + "integrity": "sha512-LFiNfRcSu7KK3evMyYOuCzv3L10TW7yC1G2/+StMjK8Y6Vqd2MG7r/Qjw4ghtuCOjFvlnms/iMmLqpvW/ES/WQ==", + "dev": true }, "repeat-string": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==" + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "dev": true }, "request": { "version": "2.88.2", @@ -19142,7 +19456,8 @@ "resolve-url": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==" + "integrity": "sha512-ZuF55hVUQaaczgOIwqWzkEcEidmlD/xl44x1UZnhOXcYuFN2S6+rcxpG+C1N3So0wvNI3DmJICUFfu2SxhBmvg==", + "dev": true }, "responselike": { "version": "1.0.2", @@ -19165,7 +19480,8 @@ "ret": { "version": "0.1.15", "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true }, "retry": { "version": "0.12.0", @@ -19242,6 +19558,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", "integrity": "sha512-ntymy489o0/QQplUDnpYAYUsO50K9SBrIVaKCWDOJzYJts0f9WH9RFJkyagebkw5+y1oi00R7ynNW/d12GBumg==", + "dev": true, "requires": { "aproba": "^1.1.1" } @@ -19264,6 +19581,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha512-aJXcif4xnaNUzvUuC5gcb46oTS7zvg4jpMTnuqtrEPlR3vFr4pxtdTwaF1Qs3Enjn9HK+ZlwQui+a7z0SywIzg==", + "dev": true, "requires": { "ret": "~0.1.10" } @@ -19382,8 +19700,7 @@ "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", - "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", - "dev": true + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, "saxes": { "version": "3.1.11", @@ -19559,6 +19876,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "dev": true, "requires": { "randombytes": "^2.1.0" } @@ -19640,6 +19958,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, "requires": { "extend-shallow": "^2.0.1", "is-extendable": "^0.1.1", @@ -19651,6 +19970,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -19659,6 +19979,7 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, "requires": { "isobject": "^3.0.1" } @@ -19668,7 +19989,8 @@ "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true }, "setprototypeof": { "version": "1.2.0", @@ -19850,6 +20172,7 @@ "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, "requires": { "base": "^0.11.1", "debug": "^2.2.0", @@ -19865,6 +20188,7 @@ "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, "requires": { "ms": "2.0.0" } @@ -19873,6 +20197,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -19881,6 +20206,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, "requires": { "is-extendable": "^0.1.0" } @@ -19888,12 +20214,14 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==" + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true } } }, @@ -19901,6 +20229,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, "requires": { "define-property": "^1.0.0", "isobject": "^3.0.0", @@ -19911,6 +20240,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", "integrity": "sha512-cZTYKFWspt9jZsMscWo8sc/5lbPC9Q0N5nBLgb+Yd915iL3udB1uFgS3B8YCx66UVHq018DAVFoee7x+gxggeA==", + "dev": true, "requires": { "is-descriptor": "^1.0.0" } @@ -19919,6 +20249,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -19927,6 +20258,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, "requires": { "kind-of": "^6.0.0" } @@ -19935,6 +20267,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, "requires": { "is-accessor-descriptor": "^1.0.0", "is-data-descriptor": "^1.0.0", @@ -19947,6 +20280,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, "requires": { "kind-of": "^3.2.0" }, @@ -19955,6 +20289,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -20065,6 +20400,7 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "dev": true, "requires": { "atob": "^2.1.2", "decode-uri-component": "^0.2.0", @@ -20077,6 +20413,7 @@ "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, "requires": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" @@ -20085,7 +20422,8 @@ "source-map-url": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.1.tgz", - "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==" + "integrity": "sha512-cPiFOTLUKvJFIg4SKVScy4ilPPW6rFgMgfuZJPNoDuMs3nC1HbMUycBoJw77xFIp6z1UJQJOfx6C9GMH80DiTw==", + "dev": true }, "sourcemap-codec": { "version": "1.4.8", @@ -20169,10 +20507,16 @@ } } }, + "split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==" + }, "split-string": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, "requires": { "extend-shallow": "^3.0.0" } @@ -20248,6 +20592,7 @@ "version": "0.1.2", "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", "integrity": "sha512-72E9+uLc27Mt718pMHt9VMNiAL4LMsmDbBva8mxWUCkT07fSzEGMYUCk0XWY6lp0j6RBAG4cJ3mWuZv2OE3s0g==", + "dev": true, "requires": { "define-property": "^0.2.5", "object-copy": "^0.1.0" @@ -20257,6 +20602,7 @@ "version": "0.2.5", "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", "integrity": "sha512-Rr7ADjQZenceVOAKop6ALkkRAmH1A4Gx9hV/7ZujPUN2rkATqFO0JZLZInbAjpZYoJ1gUx8MRMQVkYemcbMSTA==", + "dev": true, "requires": { "is-descriptor": "^0.1.0" } @@ -20288,6 +20634,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "dev": true, "requires": { "inherits": "~2.0.1", "readable-stream": "^2.0.2" @@ -20297,6 +20644,7 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.3.tgz", "integrity": "sha512-vlMC2f8I2u/bZGqkdfLQW/13Zihpej/7PmSiMQsbYddxuTsJp8vRe2x2FvVExZg7FaOds43ROAuFJwPR4MTZLw==", + "dev": true, "requires": { "end-of-stream": "^1.1.0", "stream-shift": "^1.0.0" @@ -20306,6 +20654,7 @@ "version": "2.8.3", "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz", "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==", + "dev": true, "requires": { "builtin-status-codes": "^3.0.0", "inherits": "^2.0.1", @@ -20317,7 +20666,8 @@ "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", - "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==", + "dev": true }, "streamsearch": { "version": "0.1.2", @@ -20331,14 +20681,6 @@ "integrity": "sha512-R3f198pcvnB+5IpnBlRkphuE9n46WyVl8I39W/ZUTZLz4nqSP/oLYUrcnJrw462Ds8he4YKMov2efsTIw1BDGQ==", "dev": true }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, "string-convert": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/string-convert/-/string-convert-0.2.1.tgz", @@ -20403,6 +20745,14 @@ "es-abstract": "^1.19.5" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -20458,6 +20808,11 @@ "escape-string-regexp": "^1.0.2" } }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "stylehacks": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/stylehacks/-/stylehacks-4.0.3.tgz", @@ -20676,7 +21031,8 @@ "tapable": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", - "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==" + "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", + "dev": true }, "tar": { "version": "6.1.11", @@ -20861,6 +21217,7 @@ "version": "4.8.0", "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", + "dev": true, "requires": { "commander": "^2.20.0", "source-map": "~0.6.1", @@ -20870,7 +21227,8 @@ "commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true } } }, @@ -20878,6 +21236,7 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz", "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==", + "dev": true, "requires": { "cacache": "^12.0.2", "find-cache-dir": "^2.1.0", @@ -20894,6 +21253,7 @@ "version": "12.0.4", "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", + "dev": true, "requires": { "bluebird": "^3.5.5", "chownr": "^1.1.1", @@ -20915,12 +21275,14 @@ "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true }, "lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, "requires": { "yallist": "^3.0.2" } @@ -20929,6 +21291,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -20937,6 +21300,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -20945,6 +21309,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dev": true, "requires": { "ajv": "^6.1.0", "ajv-errors": "^1.0.0", @@ -20955,6 +21320,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz", "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", + "dev": true, "requires": { "figgy-pudding": "^3.5.1" } @@ -20963,6 +21329,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dev": true, "requires": { "source-list-map": "^2.0.0", "source-map": "~0.6.1" @@ -20971,12 +21338,14 @@ "y18n": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", - "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==" + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", + "dev": true }, "yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true } } }, @@ -21065,6 +21434,7 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, "requires": { "readable-stream": "~2.3.6", "xtend": "~4.0.1" @@ -21086,6 +21456,7 @@ "version": "2.0.12", "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==", + "dev": true, "requires": { "setimmediate": "^1.0.4" } @@ -21119,7 +21490,8 @@ "to-arraybuffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", - "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==" + "integrity": "sha512-okFlQcoGTi4LQBG/PgSYblw9VOyptsz2KJZqc6qtgGdes8VktzUQkj4BI2blit072iS8VODNcMA+tvnS9dnuMA==", + "dev": true }, "to-buffer": { "version": "1.1.1", @@ -21137,6 +21509,7 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==", + "dev": true, "requires": { "kind-of": "^3.0.2" }, @@ -21145,6 +21518,7 @@ "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "dev": true, "requires": { "is-buffer": "^1.1.5" } @@ -21160,6 +21534,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, "requires": { "define-property": "^2.0.2", "extend-shallow": "^3.0.2", @@ -21171,6 +21546,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", "integrity": "sha512-ZZWNfCjUokXXDGXFpZehJIkZqq91BcULFq/Pi7M5i4JnxXdhMKAK682z8bCW3o8Hj1wuuzoKcW3DfVzaP6VuNg==", + "dev": true, "requires": { "is-number": "^3.0.0", "repeat-string": "^1.6.1" @@ -21332,7 +21708,8 @@ "tty-browserify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", - "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==" + "integrity": "sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw==", + "dev": true }, "tunnel-agent": { "version": "0.6.0", @@ -21374,7 +21751,8 @@ "typedarray": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" + "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==", + "dev": true }, "typedarray-to-buffer": { "version": "3.1.5", @@ -21591,6 +21969,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, "requires": { "arr-union": "^3.1.0", "get-value": "^2.0.6", @@ -21656,6 +22035,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", "integrity": "sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==", + "dev": true, "requires": { "has-value": "^0.3.1", "isobject": "^3.0.0" @@ -21665,6 +22045,7 @@ "version": "0.3.1", "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", "integrity": "sha512-gpG936j8/MzaeID5Yif+577c17TxaDmhuyVgSwtnL/q8UUTySg8Mecb+8Cf1otgLoD7DDH75axp86ER7LFsf3Q==", + "dev": true, "requires": { "get-value": "^2.0.3", "has-values": "^0.1.4", @@ -21675,6 +22056,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", "integrity": "sha512-+OUdGJlgjOBZDfxnDjYYG6zp487z0JGNQq3cYQYg5f5hKR+syHMsaztzGeml/4kGG55CSpKSpWTY+jYGgsHLgA==", + "dev": true, "requires": { "isarray": "1.0.0" } @@ -21684,7 +22066,8 @@ "has-values": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", - "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==" + "integrity": "sha512-J8S0cEdWuQbqD9//tlZxiMuMNmxB8PlEwvYwuxsTmR1G5RXUePEX/SJn7aD0GMLieuZYSwNH0cQuJGwnYunXRQ==", + "dev": true } } }, @@ -21692,7 +22075,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", - "devOptional": true + "dev": true }, "update-browserslist-db": { "version": "1.0.4", @@ -21760,12 +22143,14 @@ "urix": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==" + "integrity": "sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==", + "dev": true }, "url": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", "integrity": "sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ==", + "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -21774,7 +22159,8 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==", + "dev": true } } }, @@ -21844,12 +22230,14 @@ "use": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==" + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true }, "util": { "version": "0.11.1", "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dev": true, "requires": { "inherits": "2.0.3" }, @@ -21857,7 +22245,8 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==" + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true } } }, @@ -21952,7 +22341,8 @@ "vm-browserify": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", - "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", + "dev": true }, "vue": { "version": "3.2.37", @@ -21969,8 +22359,7 @@ "vue-chartjs": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/vue-chartjs/-/vue-chartjs-4.1.2.tgz", - "integrity": "sha512-QSggYjeFv/L4jFSBQpX8NzrAvX0B+Ha6nDgxkTG8tEXxYOOTwKI4phRLe+B4f+REnkmg7hgPY24R0cixZJyXBg==", - "requires": {} + "integrity": "sha512-QSggYjeFv/L4jFSBQpX8NzrAvX0B+Ha6nDgxkTG8tEXxYOOTwKI4phRLe+B4f+REnkmg7hgPY24R0cixZJyXBg==" }, "vue-clipboard2": { "version": "0.3.3", @@ -22406,8 +22795,7 @@ "vue-web-storage": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/vue-web-storage/-/vue-web-storage-6.1.0.tgz", - "integrity": "sha512-Qsa6QkUyGP+Tj0oxLRc6vEATv6axY89LbwfX8eaMM3i7K/Nl9m0NTELtNp0s8Xhg9F0SkeEP4NC3+LQL3ygMQw==", - "requires": {} + "integrity": "sha512-Qsa6QkUyGP+Tj0oxLRc6vEATv6axY89LbwfX8eaMM3i7K/Nl9m0NTELtNp0s8Xhg9F0SkeEP4NC3+LQL3ygMQw==" }, "vue3-clipboard": { "version": "1.0.0", @@ -22492,6 +22880,7 @@ "version": "1.7.5", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.5.tgz", "integrity": "sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ==", + "dev": true, "requires": { "chokidar": "^3.4.1", "graceful-fs": "^4.1.2", @@ -22503,6 +22892,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.1.tgz", "integrity": "sha512-nCFfBIPKr5Sh61s4LPpy1Wtfi0HE8isJ3d2Yb5/Ppw2P2B/3eVSEBjKfN0fmHJSK14+31KwMKmcrzs2GM4P0Ww==", + "dev": true, "optional": true, "requires": { "chokidar": "^2.1.8" @@ -22512,6 +22902,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "dev": true, "optional": true, "requires": { "micromatch": "^3.1.4", @@ -22522,6 +22913,7 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", "integrity": "sha512-3pKJwH184Xo/lnH6oyP1q2pMd7HcypqqmRs91/6/i2CGtWwIKGCkOOMTm/zXbgTEWHw1uNpNi/igc3ePOYHb6w==", + "dev": true, "optional": true, "requires": { "remove-trailing-separator": "^1.0.1" @@ -22533,12 +22925,14 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "dev": true, "optional": true }, "chokidar": { "version": "2.1.8", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "dev": true, "optional": true, "requires": { "anymatch": "^2.0.0", @@ -22559,6 +22953,7 @@ "version": "1.2.13", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.13.tgz", "integrity": "sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==", + "dev": true, "optional": true, "requires": { "bindings": "^1.5.0", @@ -22569,6 +22964,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", "integrity": "sha512-9fRVlXc0uCxEDj1nQzaWONSpbTfx0FmJfzHF7pwlI8DkWGoHBBea4Pg5Ky0ojwwxQmnSifgbKkI06Qv0Ljgj+Q==", + "dev": true, "optional": true, "requires": { "binary-extensions": "^1.0.0" @@ -22578,6 +22974,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "dev": true, "optional": true, "requires": { "graceful-fs": "^4.1.11", @@ -22605,6 +23002,29 @@ "defaults": "^1.0.3" } }, + "web-encoding": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", + "integrity": "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA==", + "requires": { + "@zxing/text-encoding": "0.9.0", + "util": "^0.12.3" + }, + "dependencies": { + "util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "requires": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + } + } + }, "webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", @@ -22615,6 +23035,7 @@ "version": "4.46.0", "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.46.0.tgz", "integrity": "sha512-6jJuJjg8znb/xRItk7bkT0+Q7AHCYjjFnvKIWQPkNIOyRqoCGvkOs0ipeQzrqz4l5FtN5ZI/ukEHroeX/o1/5Q==", + "dev": true, "requires": { "@webassemblyjs/ast": "1.9.0", "@webassemblyjs/helper-module-context": "1.9.0", @@ -22645,6 +23066,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, "requires": { "minimist": "^1.2.0" } @@ -22653,6 +23075,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "dev": true, "requires": { "big.js": "^5.2.2", "emojis-list": "^3.0.0", @@ -22663,6 +23086,7 @@ "version": "0.5.6", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, "requires": { "minimist": "^1.2.6" } @@ -22671,6 +23095,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dev": true, "requires": { "ajv": "^6.1.0", "ajv-errors": "^1.0.0", @@ -22681,6 +23106,7 @@ "version": "1.4.3", "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dev": true, "requires": { "source-list-map": "^2.0.0", "source-map": "~0.6.1" @@ -23315,6 +23741,18 @@ "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", "dev": true }, + "which-typed-array": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", + "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", + "requires": { + "available-typed-arrays": "^1.0.5", + "call-bind": "^1.0.2", + "for-each": "^0.3.3", + "gopd": "^1.0.1", + "has-tostringtag": "^1.0.0" + } + }, "wide-align": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", @@ -23371,6 +23809,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", + "dev": true, "requires": { "errno": "~0.1.7" } @@ -23460,14 +23899,18 @@ "version": "7.5.8", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.8.tgz", "integrity": "sha512-ri1Id1WinAX5Jqn9HejiGb8crfRio0Qgu8+MtL36rlTA6RLsMdWt1Az/19A2Qij6uSHUMphEFaTKa4WG+UNHNw==", - "dev": true, - "requires": {} + "dev": true }, "xdg-basedir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==" }, + "xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==" + }, "xml-name-validator": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", @@ -23478,7 +23921,6 @@ "version": "0.4.23", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", - "dev": true, "requires": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" @@ -23487,8 +23929,7 @@ "xmlbuilder": { "version": "11.0.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "dev": true + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" }, "xmlchars": { "version": "2.2.0", @@ -23517,7 +23958,8 @@ "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true }, "y18n": { "version": "5.0.8", diff --git a/ui/package.json b/ui/package.json index 1c6e42a321e..fd9eeb72058 100644 --- a/ui/package.json +++ b/ui/package.json @@ -53,6 +53,7 @@ "js-cookie": "^2.2.1", "lodash": "^4.17.15", "md5": "^2.2.1", + "minio": "^7.0.33", "mitt": "^2.1.0", "moment": "^2.26.0", "moment-timezone": "^0.5.43", diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index 17da726ae45..de01e4b9282 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -25,6 +25,8 @@ "label.access": "Access", "label.access.kubernetes.nodes": "Access Kubernetes nodes", "label.accesskey": "Access key", +"label.access.key": "Access key", +"label.secret.key": "Secret key", "label.account": "Account", "label.account.and.security.group": "Account - security group", "label.account.id": "Account ID", @@ -1182,6 +1184,7 @@ "label.limitcpuuse": "CPU cap", "label.limits": "Limits", "label.limits.configure": "Configure limits", +"label.link": "Link", "label.link.domain.to.ldap": "Link domain to LDAP", "label.linklocalip": "Link-local/Control IP address", "label.linux": "Linux", @@ -1278,6 +1281,9 @@ "label.memused": "Memory usage", "label.menu.security.groups": "Security groups", "label.menu.service.offerings": "Service offerings", +"label.metadata": "Metadata", +"label.metadata.description": "Metadata of the Object", +"label.metadata.upload.description": "Set metadata for the object", "label.metrics": "Metrics", "label.migrate.allowed": "Migrate allowed", "label.migrate.data.from.image.store": "Migrate data from image store", @@ -1407,6 +1413,16 @@ "label.oauth.configuration": "OAuth configuration", "label.oauth.verification": "OAuth verification", "label.ocfs2": "OCFS2", +"label.object.storage" : "Object Storage", +"label.object.presigned.url": "Presigned URL", +"label.object.presigned.url.description" : "Presigned URL of the object in order to access it without authentication.", +"label.object.url.description" : "URL of the object", +"label.objectstore" : "Object Storage", +"label.objectstore.search" : "Prefix based search in current directory", +"label.add.object.storage" : "Add Object Storage", +"label.add.key.value": "Add key value pair", +"label.action.update.object.storage" : "Update Object Storage", +"label.action.delete.object.storage" : "Delete Object Storage", "label.of": "of", "label.of.month": "of month", "label.offerha": "Offer HA", @@ -2119,6 +2135,8 @@ "label.updating": "Updating", "label.upgrade.router.newer.template": "Upgrade router to use newer Template", "label.upload": "Upload", +"label.upload.description": "Path to upload objects at", +"label.upload.path": "Upload path", "label.upload.icon": "Upload icon", "label.upload.iso.from.local": "Upload ISO from local", "label.upload.resource.icon": "Upload icon", @@ -2318,6 +2336,17 @@ "label.zonenamelabel": "Zone name", "label.zones": "Zones", "label.zonewizard.traffictype.storage": "Storage: Traffic between primary and secondary storage servers, such as Instance Templates and Snapshots.", +"label.buckets": "Buckets", +"label.objectstorageid": "Object Storage Pool", +"label.bucket.update": "Update Bucket", +"label.bucket.delete": "Delete Bucket", +"label.quotagb": "Quota in GB", +"label.encryption": "Encryption", +"label.versioning": "Versioning", +"label.objectlocking": "Object Lock", +"label.bucket.policy": "Bucket Policy", +"label.usersecretkey": "Secret Key", +"label.create.bucket": "Create Bucket", "message.acquire.ip.failed": "Failed to acquire IP.", "message.action.acquire.ip": "Please confirm that you want to acquire new IP.", "message.action.cancel.maintenance": "Your host has been successfully canceled for maintenance. This process can take up to several minutes.", @@ -2554,6 +2583,8 @@ "message.confirm.type": "To confirm, please type", "message.confirm.upgrade.router.newer.template": "Please confirm that you want to upgrade router to use newer Template.", "message.cpu.usage.info": "The CPU usage percentage can exceed 100% if the Instance has more than 1 vCPU or when CPU Cap is not enabled. This behavior happens according to the hypervisor being used (e.g: in KVM), due to how they account the stats", +"message.create.bucket.failed": "Failed to create bucket.", +"message.create.bucket.processing": "Bucket creation in progress", "message.create.compute.offering": "Compute offering created", "message.create.tungsten.public.network": "Create Tungsten-Fabric public Network", "message.create.internallb": "Creating internal LB", @@ -3038,6 +3069,7 @@ "message.success.add.network.static.route": "Successfully added Network Static Route", "message.success.add.network.permissions": "Successfully added Network permissions", "message.success.add.physical.network": "Successfully added Physical Network", +"message.success.add.object.storage": "Successfully added Object Storage", "message.success.add.policy.rule": "Successfully added Policy rule", "message.success.add.port.forward": "Successfully added new port forwarding rule", "message.success.add.private.gateway": "Successfully added private gateway", @@ -3066,6 +3098,7 @@ "message.success.config.vm.schedule": "Successfully configured Instance schedule", "message.success.copy.clipboard": "Successfully copied to clipboard", "message.success.create.account": "Successfully created Account", +"message.success.create.bucket": "Successfully created bucket", "message.success.create.internallb": "Successfully created Internal Load Balancer", "message.success.create.isolated.network": "Successfully created isolated Network", "message.success.create.keypair": "Successfully created SSH key pair", @@ -3106,6 +3139,8 @@ "message.success.register.user.data": "Successfully registered Userdata", "message.success.release.ip": "Successfully released IP", "message.success.remove.egress.rule": "Successfully removed egress rule", +"message.success.remove.objectstore.objects": "Successfully removed selected object(s)", +"message.success.remove.objectstore.directory": "Successfully removed selected directory", "message.success.remove.firewall.rule": "Successfully removed firewall rule", "message.success.remove.instance.rule": "Successfully removed Instance from rule", "message.success.remove.ip": "Successfully removed IP", @@ -3124,6 +3159,7 @@ "message.success.resize.volume": "Successfully resized volume", "message.success.scale.kubernetes": "Successfully scaled Kubernetes cluster", "message.success.unmanage.instance": "Successfully unmanaged Instance", +"message.success.update.bucket": "Successfully updated bucket", "message.success.update.condition": "Successfully updated condition", "message.success.update.ipaddress": "Successfully updated IP address", "message.success.update.iprange": "Successfully updated IP range", @@ -3244,6 +3280,8 @@ "message.zone.detail.description": "Populate zone details.", "message.zone.detail.hint": "A zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", "message.validate.min": "Please enter a value greater than or equal to {0}.", +"message.action.delete.object.storage": "Please confirm that you want to delete this Object Store", +"message.bucket.delete": "Please confirm that you want to delete this Bucket", "migrate.from": "Migrate from", "migrate.to": "Migrate to", "migrationPolicy": "Migration policy", diff --git a/ui/src/components/KeyValuePairInput.vue b/ui/src/components/KeyValuePairInput.vue new file mode 100644 index 00000000000..03989de50de --- /dev/null +++ b/ui/src/components/KeyValuePairInput.vue @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/components/view/AnnotationsTab.vue b/ui/src/components/view/AnnotationsTab.vue index 97cf926dee3..5b42af8eabc 100644 --- a/ui/src/components/view/AnnotationsTab.vue +++ b/ui/src/components/view/AnnotationsTab.vue @@ -193,6 +193,7 @@ export default { case 'VirtualRouter': return 'VR' case 'AutoScaleVmGroup': return 'AUTOSCALE_VM_GROUP' case 'ManagementServer': return 'MANAGEMENT_SERVER' + case 'ObjectStorage': return 'OBJECT_STORAGE' default: return '' } }, diff --git a/ui/src/components/view/ListView.vue b/ui/src/components/view/ListView.vue index cf52f058075..53c6efb321d 100644 --- a/ui/src/components/view/ListView.vue +++ b/ui/src/components/view/ListView.vue @@ -279,6 +279,9 @@ + @@ -585,7 +588,7 @@ export default { '/volume', '/snapshot', '/vmsnapshot', '/backup', '/guestnetwork', '/vpc', '/vpncustomergateway', '/vnfapp', '/template', '/iso', - '/project', '/account', + '/project', '/account', 'buckets', 'objectstore', '/zone', '/pod', '/cluster', '/host', '/storagepool', '/imagestore', '/systemvm', '/router', '/ilbvm', '/annotation', '/computeoffering', '/systemoffering', '/diskoffering', '/backupoffering', '/networkoffering', '/vpcoffering', '/tungstenfabric', '/oauthsetting', '/guestos', '/guestoshypervisormapping'].join('|')) @@ -595,7 +598,7 @@ export default { return ['vm', 'alert', 'vmgroup', 'ssh', 'userdata', 'affinitygroup', 'autoscalevmgroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork', 'vpc', 'publicip', 'vpnuser', 'vpncustomergateway', 'vnfapp', 'project', 'account', 'systemvm', 'router', 'computeoffering', 'systemoffering', - 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment' + 'diskoffering', 'backupoffering', 'networkoffering', 'vpcoffering', 'ilbvm', 'kubernetes', 'comment', 'buckets' ].includes(this.$route.name) }, getDateAtTimeZone (date, timezone) { diff --git a/ui/src/components/view/ObjectStoreBrowser.vue b/ui/src/components/view/ObjectStoreBrowser.vue new file mode 100644 index 00000000000..f2e68c8b954 --- /dev/null +++ b/ui/src/components/view/ObjectStoreBrowser.vue @@ -0,0 +1,541 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/config/section/infra.js b/ui/src/config/section/infra.js index 41b08fc6f51..0b3b1e7ae4f 100644 --- a/ui/src/config/section/infra.js +++ b/ui/src/config/section/infra.js @@ -23,6 +23,7 @@ import clusters from '@/config/section/infra/clusters' import hosts from '@/config/section/infra/hosts' import primaryStorages from '@/config/section/infra/primaryStorages' import secondaryStorages from '@/config/section/infra/secondaryStorages' +import objectStorages from '@/config/section/infra/objectStorages' import systemVms from '@/config/section/infra/systemVms' import routers from '@/config/section/infra/routers' import ilbvms from '@/config/section/infra/ilbvms' @@ -49,6 +50,7 @@ export default { hosts, primaryStorages, secondaryStorages, + objectStorages, systemVms, routers, ilbvms, diff --git a/ui/src/config/section/infra/objectStorages.js b/ui/src/config/section/infra/objectStorages.js new file mode 100644 index 00000000000..821c1b2d948 --- /dev/null +++ b/ui/src/config/section/infra/objectStorages.js @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import { shallowRef, defineAsyncComponent } from 'vue' +import store from '@/store' + +export default { + name: 'objectstore', + title: 'label.object.storage', + icon: 'gold-outlined', + docHelp: 'adminguide/storage.html#object-storage', + permission: ['listObjectStoragePools'], + columns: () => { + var fields = ['name', 'url', 'providername'] + return fields + }, + details: () => { + var fields = ['name', 'id', 'url', 'providername'] + return fields + }, + resourceType: 'ObjectStorage', + tabs: [{ + name: 'details', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) + }, { + name: 'events', + resourceType: 'ObjectStore', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/EventsTab.vue'))), + show: () => { return 'listEvents' in store.getters.apis } + }, { + name: 'comments', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) + }], + actions: [ + { + api: 'addObjectStoragePool', + icon: 'plus-outlined', + docHelp: 'installguide/configuration.html#add-object-storage', + label: 'label.add.object.storage', + listView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/AddObjectStorage.vue'))) + }, + { + api: 'updateObjectStoragePool', + icon: 'edit-outlined', + label: 'label.action.update.object.storage', + args: ['name', 'url'], + dataView: true + }, + { + api: 'deleteObjectStoragePool', + icon: 'delete-outlined', + label: 'label.action.delete.object.storage', + message: 'message.action.delete.object.storage', + dataView: true, + displayName: (record) => { return record.name || record.displayName || record.id } + } + ] +} diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 3c2bd30db9b..0fbb930e750 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -164,8 +164,8 @@ export default { dataView: true, show: (record, store) => { return record.state === 'Ready' && (record.hypervisor !== 'KVM' || - record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || - record.hypervisor === 'KVM' && record.vmstate !== 'Running') + record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || + record.hypervisor === 'KVM' && record.vmstate !== 'Running') }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/TakeSnapshot.vue'))) @@ -178,8 +178,8 @@ export default { dataView: true, show: (record, store) => { return record.state === 'Ready' && (record.hypervisor !== 'KVM' || - record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || - record.hypervisor === 'KVM' && record.vmstate !== 'Running') + record.hypervisor === 'KVM' && record.vmstate === 'Running' && store.features.kvmsnapshotenabled || + record.hypervisor === 'KVM' && record.vmstate !== 'Running') }, popup: true, component: shallowRef(defineAsyncComponent(() => import('@/views/storage/RecurringSnapshotVolume.vue'))), @@ -250,8 +250,8 @@ export default { dataView: true, show: (record) => { return !['Destroy', 'Destroyed', 'Expunging', 'Expunged', 'Migrating', 'Uploading', 'UploadError', 'Creating'].includes(record.state) && - ((record.type === 'ROOT' && record.vmstate === 'Stopped') || - (record.type !== 'ROOT' && !record.virtualmachineid && !['Allocated', 'Uploaded'].includes(record.state))) + ((record.type === 'ROOT' && record.vmstate === 'Stopped') || + (record.type !== 'ROOT' && !record.virtualmachineid && !['Allocated', 'Uploaded'].includes(record.state))) }, args: (record, store) => { var fields = ['volumeid', 'name', 'displaytext', 'ostypeid', 'ispublic', 'isfeatured', 'isdynamicallyscalable', 'requireshvm', 'passwordenabled'] @@ -285,8 +285,8 @@ export default { dataView: true, show: (record, store) => { return ['Expunging', 'Expunged', 'UploadError'].includes(record.state) || - ['Allocated', 'Uploaded'].includes(record.state) && record.type !== 'ROOT' && !record.virtualmachineid || - ((['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) || store.features.allowuserexpungerecovervolume) && record.state === 'Destroy') + ['Allocated', 'Uploaded'].includes(record.state) && record.type !== 'ROOT' && !record.virtualmachineid || + ((['Admin', 'DomainAdmin'].includes(store.userInfo.roletype) || store.features.allowuserexpungerecovervolume) && record.state === 'Destroy') }, groupAction: true, popup: true, @@ -447,6 +447,119 @@ export default { show: (record) => { return record.state !== 'Destroyed' } } ] + }, + { + name: 'backup', + title: 'label.backup', + icon: 'cloud-upload-outlined', + permission: ['listBackups'], + columns: [{ name: (record) => { return record.virtualmachinename } }, 'virtualmachinename', 'status', 'type', 'created', 'account', 'zone'], + details: ['virtualmachinename', 'id', 'type', 'externalid', 'size', 'virtualsize', 'volumes', 'backupofferingname', 'zone', 'account', 'domain', 'created'], + actions: [ + { + api: 'restoreBackup', + icon: 'sync-outlined', + docHelp: 'adminguide/virtual_machines.html#restoring-vm-backups', + label: 'label.backup.restore', + message: 'message.backup.restore', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' } + }, + { + api: 'restoreVolumeFromBackupAndAttachToVM', + icon: 'paper-clip-outlined', + label: 'label.backup.attach.restore', + message: 'message.backup.attach.restore', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' }, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/RestoreAttachBackupVolume.vue'))) + }, + { + api: 'removeVirtualMachineFromBackupOffering', + icon: 'scissor-outlined', + label: 'label.backup.offering.remove', + message: 'message.backup.offering.remove', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' }, + args: ['forced', 'virtualmachineid'], + mapping: { + forced: { + value: (record) => { return true } + }, + virtualmachineid: { + value: (record) => { return record.virtualmachineid } + } + } + }, + { + api: 'deleteBackup', + icon: 'delete-outlined', + label: 'label.delete.backup', + message: 'message.delete.backup', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' } + } + ] + }, + { + name: 'buckets', + title: 'label.buckets', + icon: 'funnel-plot-outlined', + permission: ['listBuckets'], + columns: ['name', 'state', 'objectstore', 'size', 'account'], + details: ['id', 'name', 'state', 'objectstore', 'size', 'url', 'accesskey', 'usersecretkey', 'account', 'domain', 'created', 'quota', 'encryption', 'versioning', 'objectlocking', 'policy'], + tabs: [ + { + name: 'details', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) + }, + { + name: 'browser', + resourceType: 'Bucket', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/ObjectStoreBrowser.vue'))), + show: (record) => { return record.provider !== 'Simulator' } + + }, + { + name: 'events', + resourceType: 'Bucket', + component: shallowRef(defineAsyncComponent(() => import('@/components/view/EventsTab.vue'))), + show: () => { return 'listEvents' in store.getters.apis } + } + ], + actions: [ + { + api: 'createBucket', + icon: 'plus-outlined', + docHelp: 'installguide/configuration.html#create-bucket', + label: 'label.create.bucket', + listView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/CreateBucket.vue'))) + }, + { + api: 'updateBucket', + icon: 'edit-outlined', + docHelp: 'adminguide/object_storage.html#update-bucket', + label: 'label.bucket.update', + dataView: true, + popup: true, + component: shallowRef(defineAsyncComponent(() => import('@/views/storage/UpdateBucket.vue'))), + show: (record) => { return record.state !== 'Destroyed' } + }, + { + api: 'deleteBucket', + icon: 'delete-outlined', + label: 'label.bucket.delete', + message: 'message.bucket.delete', + dataView: true, + show: (record) => { return record.state !== 'Destroyed' }, + groupAction: true, + popup: true, + groupMap: (selection) => { return selection.map(x => { return { id: x } }) } + } + ] } ] } diff --git a/ui/src/core/lazy_lib/icons_use.js b/ui/src/core/lazy_lib/icons_use.js index 0a95facdd9d..dc960de1bad 100644 --- a/ui/src/core/lazy_lib/icons_use.js +++ b/ui/src/core/lazy_lib/icons_use.js @@ -88,6 +88,7 @@ import { FormOutlined, ForwardOutlined, FullscreenOutlined, + FunnelPlotOutlined, GatewayOutlined, GithubOutlined, GlobalOutlined, @@ -244,6 +245,7 @@ export default { app.component('FormOutlined', FormOutlined) app.component('ForwardOutlined', ForwardOutlined) app.component('FullscreenOutlined', FullscreenOutlined) + app.component('FunnelPlotOutlined', FunnelPlotOutlined) app.component('GatewayOutlined', GatewayOutlined) app.component('GithubOutlined', GithubOutlined) app.component('GlobalOutlined', GlobalOutlined) diff --git a/ui/src/views/infra/AddObjectStorage.vue b/ui/src/views/infra/AddObjectStorage.vue new file mode 100644 index 00000000000..4aacd6adc0f --- /dev/null +++ b/ui/src/views/infra/AddObjectStorage.vue @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/views/infra/InfraSummary.vue b/ui/src/views/infra/InfraSummary.vue index 8411dca373c..a35db46f760 100644 --- a/ui/src/views/infra/InfraSummary.vue +++ b/ui/src/views/infra/InfraSummary.vue @@ -187,7 +187,7 @@ export default { return { loading: true, routes: {}, - sections: ['zones', 'pods', 'clusters', 'hosts', 'storagepools', 'imagestores', 'systemvms', 'routers', 'cpusockets', 'managementservers', 'alerts', 'ilbvms', 'metrics'], + sections: ['zones', 'pods', 'clusters', 'hosts', 'storagepools', 'imagestores', 'objectstores', 'systemvms', 'routers', 'cpusockets', 'managementservers', 'alerts', 'ilbvms', 'metrics'], sslFormVisible: false, stats: {}, intermediateCertificates: [], diff --git a/ui/src/views/storage/CreateBucket.vue b/ui/src/views/storage/CreateBucket.vue new file mode 100644 index 00000000000..f95b7360e0c --- /dev/null +++ b/ui/src/views/storage/CreateBucket.vue @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/ui/src/views/storage/UpdateBucket.vue b/ui/src/views/storage/UpdateBucket.vue new file mode 100644 index 00000000000..5d68c901763 --- /dev/null +++ b/ui/src/views/storage/UpdateBucket.vue @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 21a81ad5c02..37906496059 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -16,40 +16,14 @@ // under the License. package com.cloud.usage; -import java.net.InetAddress; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TimeZone; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import javax.inject.Inject; -import javax.naming.ConfigurationException; - -import org.apache.cloudstack.quota.QuotaAlertManager; -import org.apache.cloudstack.quota.QuotaManager; -import org.apache.cloudstack.quota.QuotaStatement; -import org.apache.cloudstack.utils.usage.UsageUtils; -import org.apache.commons.collections.CollectionUtils; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.managed.context.ManagedContextRunnable; -import org.apache.cloudstack.usage.UsageTypes; - import com.cloud.alert.AlertManager; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventDetailsVO; import com.cloud.event.UsageEventVO; import com.cloud.event.dao.UsageEventDao; import com.cloud.event.dao.UsageEventDetailsDao; +import com.cloud.usage.dao.BucketStatisticsDao; +import com.cloud.usage.dao.UsageBackupDao; import com.cloud.usage.dao.UsageDao; import com.cloud.usage.dao.UsageIPAddressDao; import com.cloud.usage.dao.UsageJobDao; @@ -58,14 +32,15 @@ import com.cloud.usage.dao.UsageNetworkDao; import com.cloud.usage.dao.UsageNetworkOfferingDao; import com.cloud.usage.dao.UsagePortForwardingRuleDao; import com.cloud.usage.dao.UsageSecurityGroupDao; -import com.cloud.usage.dao.UsageBackupDao; -import com.cloud.usage.dao.UsageVMSnapshotOnPrimaryDao; import com.cloud.usage.dao.UsageStorageDao; import com.cloud.usage.dao.UsageVMInstanceDao; import com.cloud.usage.dao.UsageVMSnapshotDao; +import com.cloud.usage.dao.UsageVMSnapshotOnPrimaryDao; import com.cloud.usage.dao.UsageVPNUserDao; import com.cloud.usage.dao.UsageVmDiskDao; import com.cloud.usage.dao.UsageVolumeDao; +import com.cloud.usage.parser.BackupUsageParser; +import com.cloud.usage.parser.BucketUsageParser; import com.cloud.usage.parser.IPAddressUsageParser; import com.cloud.usage.parser.LoadBalancerUsageParser; import com.cloud.usage.parser.NetworkOfferingUsageParser; @@ -73,7 +48,6 @@ import com.cloud.usage.parser.NetworkUsageParser; import com.cloud.usage.parser.PortForwardingUsageParser; import com.cloud.usage.parser.SecurityGroupUsageParser; import com.cloud.usage.parser.StorageUsageParser; -import com.cloud.usage.parser.BackupUsageParser; import com.cloud.usage.parser.VMInstanceUsageParser; import com.cloud.usage.parser.VMSnapshotOnPrimaryParser; import com.cloud.usage.parser.VMSnapshotUsageParser; @@ -96,6 +70,32 @@ import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.quota.QuotaAlertManager; +import org.apache.cloudstack.quota.QuotaManager; +import org.apache.cloudstack.quota.QuotaStatement; +import org.apache.cloudstack.usage.UsageTypes; +import org.apache.cloudstack.utils.usage.UsageUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; +import java.net.InetAddress; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; @@ -164,6 +164,9 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna @Inject private QuotaStatement _quotaStatement; + @Inject + private BucketStatisticsDao _bucketStatisticsDao; + private String _version = null; private final Calendar _jobExecTime = Calendar.getInstance(); private int _aggregationDuration = 0; @@ -496,6 +499,7 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna Map networkStats = null; List vmDiskStats = null; Map vmDiskUsages = null; + List bucketStats = null; TransactionLegacy userTxn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); try { Long limit = Long.valueOf(500); @@ -626,6 +630,46 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna offset = new Long(offset.longValue() + limit.longValue()); } while ((vmDiskStats != null) && !vmDiskStats.isEmpty()); + // reset offset + offset = Long.valueOf(0); + + // get all the user stats to create usage records for the bucket usage + Long lastBucketStatsId = _usageDao.getLastBucketStatsId(); + if (lastBucketStatsId == null) { + lastBucketStatsId = Long.valueOf(0); + } + + SearchCriteria sc5 = _bucketStatisticsDao.createSearchCriteria(); + sc5.addAnd("id", SearchCriteria.Op.LTEQ, lastBucketStatsId); + do { + Filter filter = new Filter(BucketStatisticsVO.class, "id", true, offset, limit); + + bucketStats = _bucketStatisticsDao.search(sc5, filter); + + if ((bucketStats != null) && !bucketStats.isEmpty()) { + // now copy the accounts to cloud_usage db + _usageDao.updateBucketStats(bucketStats); + } + offset = new Long(offset.longValue() + limit.longValue()); + } while ((bucketStats != null) && !bucketStats.isEmpty()); + + // reset offset + offset = Long.valueOf(0); + + sc5 = _bucketStatisticsDao.createSearchCriteria(); + sc5.addAnd("id", SearchCriteria.Op.GT, lastBucketStatsId); + do { + Filter filter = new Filter(BucketStatisticsVO.class, "id", true, offset, limit); + + bucketStats = _bucketStatisticsDao.search(sc5, filter); + + if ((bucketStats != null) && !bucketStats.isEmpty()) { + // now copy the accounts to cloud_usage db + _usageDao.saveBucketStats(bucketStats); + } + offset = new Long(offset.longValue() + limit.longValue()); + } while ((bucketStats != null) && !bucketStats.isEmpty()); + } finally { userTxn.close(); } @@ -978,6 +1022,12 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna s_logger.debug("VM Backup usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); } } + parsed = BucketUsageParser.parse(account, currentStartDate, currentEndDate); + if (s_logger.isDebugEnabled()) { + if (!parsed) { + s_logger.debug("Bucket usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); + } + } return parsed; } diff --git a/usage/src/main/java/com/cloud/usage/parser/BucketUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/BucketUsageParser.java new file mode 100644 index 00000000000..1223c794ad5 --- /dev/null +++ b/usage/src/main/java/com/cloud/usage/parser/BucketUsageParser.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.parser; + +import com.cloud.usage.BucketStatisticsVO; +import com.cloud.usage.UsageVO; +import com.cloud.usage.dao.BucketStatisticsDao; +import com.cloud.usage.dao.UsageDao; +import com.cloud.user.AccountVO; +import org.apache.cloudstack.usage.UsageTypes; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@Component +public class BucketUsageParser { + public static final Logger s_logger = Logger.getLogger(BucketUsageParser.class.getName()); + + private static UsageDao s_usageDao; + private static BucketStatisticsDao s_bucketStatisticsDao; + + @Inject + private UsageDao _usageDao; + @Inject + private BucketStatisticsDao _bucketStatisticsDao; + + @PostConstruct + void init() { + s_usageDao = _usageDao; + s_bucketStatisticsDao = _bucketStatisticsDao; + } + + public static boolean parse(AccountVO account, Date startDate, Date endDate) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Parsing all Bucket usage events for account: " + account.getId()); + } + + if ((endDate == null) || endDate.after(new Date())) { + endDate = new Date(); + } + + List BucketStatisticsVOs = s_bucketStatisticsDao.listBy(account.getId()); + + List usageRecords = new ArrayList<>(); + for (BucketStatisticsVO bucketStatistics : BucketStatisticsVOs) { + long bucketSize = bucketStatistics.getSize(); + if(bucketSize > 0) { + UsageVO usageRecord = + new UsageVO(1L, account.getId(), account.getDomainId(), "Bucket Size", bucketSize + " bytes", + UsageTypes.BUCKET, new Double(bucketSize), bucketStatistics.getBucketId(), null, null, startDate, endDate); + usageRecords.add(usageRecord); + } + } + + s_usageDao.saveUsageRecords(usageRecords); + + return true; + } +} From 108651ad403e007f208af4703cc6adedf4a105ae Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 4 Dec 2023 13:18:49 +0530 Subject: [PATCH 18/50] api: make displaytext form upload template/iso optional (#8289) Signed-off-by: Abhishek Kumar --- .../api/command/user/iso/GetUploadParamsForIsoCmd.java | 4 ++-- .../command/user/template/GetUploadParamsForTemplateCmd.java | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java index a7a418f5a86..e1759566299 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/GetUploadParamsForIsoCmd.java @@ -29,6 +29,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; @@ -54,7 +55,6 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { @Parameter(name = ApiConstants.DISPLAY_TEXT, type = BaseCmd.CommandType.STRING, - required = true, description = "the display text of the ISO. This is usually used for display purposes.", length = 4096) private String displayText; @@ -85,7 +85,7 @@ public class GetUploadParamsForIsoCmd extends AbstractGetUploadParamsCmd { } public String getDisplayText() { - return displayText; + return StringUtils.isBlank(displayText) ? getName() : displayText; } public Boolean isFeatured() { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java index 3a9e1c8b429..ab872b84edb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java @@ -33,6 +33,7 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import com.cloud.exception.ResourceAllocationException; @@ -46,7 +47,7 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { private static final String s_name = "postuploadtemplateresponse"; - @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, required = true, description = "the display text of the template. This is usually used for display purposes.", length = 4096) + @Parameter(name = ApiConstants.DISPLAY_TEXT, type = CommandType.STRING, description = "the display text of the template. This is usually used for display purposes.", length = 4096) private String displayText; @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, required = true, description = "the target hypervisor for the template") @@ -95,7 +96,7 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { private Boolean deployAsIs; public String getDisplayText() { - return displayText; + return StringUtils.isBlank(displayText) ? getName() : displayText; } public String getHypervisor() { From b0910fc61d7b98bdfd3700f5641f3c84b2ec63bc Mon Sep 17 00:00:00 2001 From: Bryan Lima <42067040+BryanMLima@users.noreply.github.com> Date: Mon, 4 Dec 2023 05:52:32 -0300 Subject: [PATCH 19/50] Add dynamic secondary storage selection (#7659) --- .../com/cloud/storage/StorageService.java | 10 + .../apache/cloudstack/api/ApiConstants.java | 3 + .../cloudstack/api/ResponseGenerator.java | 4 + .../CreateSecondaryStorageSelectorCmd.java | 93 ++++++ .../ListSecondaryStorageSelectorsCmd.java | 63 ++++ .../RemoveSecondaryStorageSelectorCmd.java | 54 ++++ .../UpdateSecondaryStorageSelectorCmd.java | 67 ++++ .../SecondaryStorageHeuristicsResponse.java | 141 ++++++++ .../apache/cloudstack/query/QueryService.java | 4 + .../secstorage/heuristics/Heuristic.java | 40 +++ .../secstorage/heuristics/HeuristicType.java | 25 ++ .../api/storage/DataStoreManager.java | 4 + .../com/cloud/storage/StorageManager.java | 3 + .../com/cloud/template/TemplateManager.java | 3 +- .../cloudstack/secstorage/HeuristicVO.java | 125 ++++++++ .../dao/SecondaryStorageHeuristicDao.java | 26 ++ .../dao/SecondaryStorageHeuristicDaoImpl.java | 50 +++ .../storage/datastore/db/ImageStoreDao.java | 2 + .../datastore/db/ImageStoreDaoImpl.java | 15 + ...spring-engine-schema-core-daos-context.xml | 2 + .../META-INF/db/schema-41810to41900.sql | 15 + .../ImageStoreProviderManagerImpl.java | 12 +- .../storage/snapshot/SnapshotServiceImpl.java | 22 +- .../snapshot/SnapshotServiceImplTest.java | 57 +++- .../datastore/DataStoreManagerImpl.java | 10 + .../datastore/ImageStoreProviderManager.java | 3 + .../java/com/cloud/api/ApiResponseHelper.java | 12 + .../com/cloud/api/query/QueryManagerImpl.java | 39 +++ .../cloud/server/ManagementServerImpl.java | 9 + .../com/cloud/storage/StorageManagerImpl.java | 71 ++++ .../cloud/storage/VolumeApiServiceImpl.java | 6 +- .../template/HypervisorTemplateAdapter.java | 266 ++++++++------- .../cloud/template/TemplateManagerImpl.java | 19 +- .../java/com/cloud/test/TestAppender.java | 2 + .../heuristics/HeuristicRuleHelper.java | 278 ++++++++++++++++ .../heuristics/presetvariables/Account.java | 41 +++ .../heuristics/presetvariables/Domain.java | 30 ++ .../GenericHeuristicPresetVariable.java | 43 +++ .../presetvariables/PresetVariables.java | 72 +++++ .../presetvariables/SecondaryStorage.java | 64 ++++ .../heuristics/presetvariables/Snapshot.java | 44 +++ .../heuristics/presetvariables/Template.java | 56 ++++ .../heuristics/presetvariables/Volume.java | 44 +++ .../HypervisorTemplateAdapterTest.java | 303 +++++++++++++++++- .../template/TemplateManagerImplTest.java | 209 +++++++----- .../heuristics/HeuristicRuleHelperTest.java | 205 ++++++++++++ 46 files changed, 2456 insertions(+), 210 deletions(-) create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java create mode 100644 api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java create mode 100644 api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java create mode 100644 api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java create mode 100644 engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java create mode 100644 server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java create mode 100644 server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index 3c6c22f0ea2..c3609cfd8ee 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -36,6 +36,10 @@ import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceInUseException; import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.object.ObjectStore; public interface StorageService { @@ -112,6 +116,12 @@ public interface StorageService { StoragePool syncStoragePool(SyncStoragePoolCmd cmd); + Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd); + + Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd); + + void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd); + ObjectStore discoverObjectStore(String name, String url, String providerName, Map details) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; boolean deleteObjectStore(DeleteObjectStoragePoolCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index fa0a24670bd..ff188684557 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -1061,6 +1061,9 @@ public class ApiConstants { public static final String HAS_RULES = "hasrules"; public static final String OBJECT_STORAGE = "objectstore"; + public static final String HEURISTIC_RULE = "heuristicrule"; + public static final String HEURISTIC_TYPE_VALID_OPTIONS = "Valid options are: ISO, SNAPSHOT, TEMPLATE and VOLUME."; + public static final String MANAGEMENT = "management"; public static final String IS_VNF = "isvnf"; public static final String VNF_NICS = "vnfnics"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 944bb2194a5..0bddf6d2994 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -104,6 +104,7 @@ import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; @@ -148,6 +149,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.usage.Usage; @@ -536,6 +538,8 @@ public interface ResponseGenerator { FirewallResponse createIpv6FirewallRuleResponse(FirewallRule acl); + SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic); + IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine publicIp); ObjectStoreResponse createObjectStoreResponse(ObjectStore os); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..a0e99b55971 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/CreateSecondaryStorageSelectorCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "createSecondaryStorageSelector", description = "Creates a secondary storage selector, described by the heuristic rule.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, entityType = {Heuristic.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class CreateSecondaryStorageSelectorCmd extends BaseCmd{ + + @Parameter(name = ApiConstants.NAME, required = true, type = CommandType.STRING, description = "The name identifying the heuristic rule.") + private String name; + + @Parameter(name = ApiConstants.DESCRIPTION, required = true, type = BaseCmd.CommandType.STRING, description = "The description of the heuristic rule.") + private String description; + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = BaseCmd.CommandType.UUID, description = "The zone in which the heuristic " + + "rule will be applied.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, required = true, type = BaseCmd.CommandType.STRING, description = + "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.createSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create a secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java new file mode 100644 index 00000000000..c437cd660e7 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/ListSecondaryStorageSelectorsCmd.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@APICommand(name = "listSecondaryStorageSelectors", description = "Lists the secondary storage selectors and their rules.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class ListSecondaryStorageSelectorsCmd extends BaseListCmd { + + @Parameter(name = ApiConstants.ZONE_ID, required = true, entityType = ZoneResponse.class, type = CommandType.UUID, description = "The zone ID to be used in the search filter.") + private Long zoneId; + + @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = + "Whether to filter the selectors by type and, if so, which one. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @Parameter(name = ApiConstants.SHOW_REMOVED, type = CommandType.BOOLEAN, description = "Show removed heuristics.") + private boolean showRemoved = false; + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public boolean isShowRemoved() { + return showRemoved; + } + + @Override + public void execute() { + ListResponse response = _queryService.listSecondaryStorageSelectors(this); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..79554f44782 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/RemoveSecondaryStorageSelectorCmd.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "removeSecondaryStorageSelector", description = "Removes an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class RemoveSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector to be removed.") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() { + _storageService.removeSecondaryStorageHeuristic(this); + final SuccessResponse response = new SuccessResponse(); + response.setResponseName(getCommandName()); + response.setSuccess(true); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java new file mode 100644 index 00000000000..e63d1cbfa27 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/heuristics/UpdateSecondaryStorageSelectorCmd.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage.heuristics; + +import com.cloud.user.Account; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +@APICommand(name = "updateSecondaryStorageSelector", description = "Updates an existing secondary storage selector.", since = "4.19.0", responseObject = + SecondaryStorageHeuristicsResponse.class, requestHasSensitiveInfo = false, entityType = {Heuristic.class}, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}) +public class UpdateSecondaryStorageSelectorCmd extends BaseCmd { + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, entityType = SecondaryStorageHeuristicsResponse.class, required = true, + description = "The unique identifier of the secondary storage selector.") + private Long id; + + @Parameter(name = ApiConstants.HEURISTIC_RULE, required = true, type = BaseCmd.CommandType.STRING, description = "The heuristic rule, in JavaScript language. It is required " + + "that it returns the UUID of a secondary storage pool. An example of a rule is `if (snapshot.hypervisorType === 'KVM') { '7832f261-c602-4e8e-8580-2496ffbbc45d'; " + + "}` would allocate all snapshots with the KVM hypervisor to the specified secondary storage UUID.", length = 65535) + private String heuristicRule; + + public Long getId() { + return id; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + @Override + public void execute() { + Heuristic heuristic = _storageService.updateSecondaryStorageHeuristic(this); + + if (heuristic == null) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update the secondary storage selector."); + } + + SecondaryStorageHeuristicsResponse response = _responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java new file mode 100644 index 00000000000..25a6b2eca75 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/SecondaryStorageHeuristicsResponse.java @@ -0,0 +1,141 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import org.apache.cloudstack.api.EntityReference; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; + +import java.util.Date; + +import static org.apache.cloudstack.api.ApiConstants.HEURISTIC_TYPE_VALID_OPTIONS; + +@EntityReference(value = {Heuristic.class}) +public class SecondaryStorageHeuristicsResponse extends BaseResponse { + + @SerializedName(ApiConstants.ID) + @Param(description = "ID of the heuristic.") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the heuristic.") + private String name; + + @SerializedName(ApiConstants.DESCRIPTION) + @Param(description = "Description of the heuristic.") + private String description; + + @SerializedName(ApiConstants.ZONE_ID) + @Param(description = "The zone which the heuristic is valid upon.") + private String zoneId; + + @SerializedName(ApiConstants.TYPE) + @Param(description = "The resource type directed to a specific secondary storage by the selector. " + HEURISTIC_TYPE_VALID_OPTIONS) + private String type; + + @SerializedName(ApiConstants.HEURISTIC_RULE) + @Param(description = "The heuristic rule, in JavaScript language, used to select a secondary storage to be directed.") + private String heuristicRule; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "When the heuristic was created.") + private Date created; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "When the heuristic was removed.") + private Date removed; + + + public SecondaryStorageHeuristicsResponse(String id, String name, String description, String zoneId, String type, String heuristicRule, Date created, Date removed) { + super("heuristics"); + this.id = id; + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + this.created = created; + this.removed = removed; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getZoneId() { + return zoneId; + } + + public void setZoneId(String zoneId) { + this.zoneId = zoneId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + public Date getCreated() { + return created; + } + + public void setCreated(Date created) { + this.created = created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 17dd14da84c..3299e7537a2 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.user.account.ListAccountsCmd; import org.apache.cloudstack.api.command.user.account.ListProjectAccountsCmd; @@ -79,6 +80,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -189,6 +191,8 @@ public interface QueryService { List listRouterHealthChecks(GetRouterHealthCheckResultsCmd cmd); + ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd); + ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd); ListResponse listSnapshots(ListSnapshotsCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java new file mode 100644 index 00000000000..2a0b8d6ea24 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/Heuristic.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +import org.apache.cloudstack.api.Identity; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface Heuristic extends InternalIdentity, Identity { + + String getName(); + + String getDescription(); + + Long getZoneId(); + + String getType(); + + String getHeuristicRule(); + + Date getCreated(); + + Date getRemoved(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java new file mode 100644 index 00000000000..f23e4b0b633 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/secstorage/heuristics/HeuristicType.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.heuristics; + +/** + * The type of the heuristic used in the allocation process of secondary storage resources. + * Valid options are: {@link #ISO}, {@link #SNAPSHOT}, {@link #TEMPLATE} and {@link #VOLUME} + */ +public enum HeuristicType { + ISO, SNAPSHOT, TEMPLATE, VOLUME +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 3ee5803a91a..de26a09c6e5 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -35,6 +35,8 @@ public interface DataStoreManager { List getImageStoresByScopeExcludingReadOnly(ZoneScope scope); + List getImageStoresByZoneIds(Long ... zoneIds); + DataStore getRandomImageStore(long zoneId); DataStore getRandomUsableImageStore(long zoneId); @@ -55,5 +57,7 @@ public interface DataStoreManager { boolean isRegionStore(DataStore store); + DataStore getImageStoreByUuid(String uuid); + Long getStoreZoneId(long storeId, DataStoreRole role); } diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 28e7c89419a..1437457725a 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -192,6 +192,9 @@ public interface StorageManager extends StorageService { ConfigKey.Scope.Global, null); + ConfigKey HEURISTICS_SCRIPT_TIMEOUT = new ConfigKey<>("Advanced", Long.class, "heuristics.script.timeout", "3000", + "The maximum runtime, in milliseconds, to execute the heuristic rule; if it is reached, a timeout will happen.", true); + /** * should we execute in sequence not involving any storages? * @return tru if commands should execute in sequence diff --git a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java index 2c129dfd6a5..3b3537d5488 100644 --- a/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java +++ b/engine/components-api/src/main/java/com/cloud/template/TemplateManager.java @@ -34,6 +34,7 @@ import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachineProfile; @@ -116,7 +117,7 @@ public interface TemplateManager { Long getTemplateSize(long templateId, long zoneId); - DataStore getImageStore(String storeUuid, Long zoneId); + DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume); String getChecksum(DataStore store, String templatePath, String algorithm); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java new file mode 100644 index 00000000000..b0da0c5e747 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/HeuristicVO.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.util.Date; +import java.util.UUID; + +@Entity +@Table(name = "heuristics") +public class HeuristicVO implements Heuristic { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private Long id; + + @Column(name = "uuid", nullable = false) + private String uuid = UUID.randomUUID().toString(); + + @Column(name = "name") + private String name; + + @Column(name = "description") + private String description; + + @Column(name = "zone_id", nullable = false) + private Long zoneId; + + @Column(name = "type", nullable = false) + private String type; + + @Column(name = "heuristic_rule", nullable = false, length = 65535) + private String heuristicRule; + + @Column(name = GenericDao.CREATED_COLUMN, nullable = false) + @Temporal(value = TemporalType.TIMESTAMP) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + public HeuristicVO() { + } + + public HeuristicVO(String name, String description, Long zoneId, String type, String heuristicRule) { + this.name = name; + this.description = description; + this.zoneId = zoneId; + this.type = type; + this.heuristicRule = heuristicRule; + } + + @Override + public long getId() { + return id; + } + + public String getUuid() { + return uuid; + } + + @Override + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public Long getZoneId() { + return zoneId; + } + + public String getType() { + return type; + } + + public String getHeuristicRule() { + return heuristicRule; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setHeuristicRule(String heuristicRule) { + this.heuristicRule = heuristicRule; + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "name", "heuristicRule", "type"); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java new file mode 100644 index 00000000000..1c4057a7beb --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDao.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; + +public interface SecondaryStorageHeuristicDao extends GenericDao { + + HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java new file mode 100644 index 00000000000..0b51b2aec6b --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/secstorage/dao/SecondaryStorageHeuristicDaoImpl.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.secstorage.dao; + +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; + +@Component +public class SecondaryStorageHeuristicDaoImpl extends GenericDaoBase implements SecondaryStorageHeuristicDao { + private SearchBuilder zoneAndTypeSearch; + + @PostConstruct + public void init() { + zoneAndTypeSearch = createSearchBuilder(); + zoneAndTypeSearch.and("zoneId", zoneAndTypeSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + zoneAndTypeSearch.and("type", zoneAndTypeSearch.entity().getType(), SearchCriteria.Op.IN); + zoneAndTypeSearch.done(); + } + + @Override + public HeuristicVO findByZoneIdAndType(long zoneId, HeuristicType type) { + SearchCriteria searchCriteria = zoneAndTypeSearch.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParameters("type", type.toString()); + final Filter filter = new Filter(HeuristicVO.class, "created", false); + + return findOneBy(searchCriteria, filter); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index ba9825c3c86..1c31b3e0cc4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -49,4 +49,6 @@ public interface ImageStoreDao extends GenericDao { List findByProtocol(String protocol); ImageStoreVO findOneByZoneAndProtocol(long zoneId, String protocol); + + List listImageStoresByZoneIds(Long... zoneIds); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java index 3468b6008d9..a4827a1beae 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreDaoImpl.java @@ -43,6 +43,8 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem private SearchBuilder protocolSearch; private SearchBuilder zoneProtocolSearch; + private SearchBuilder zonesInSearch; + public ImageStoreDaoImpl() { super(); protocolSearch = createSearchBuilder(); @@ -55,6 +57,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem zoneProtocolSearch.and("protocol", zoneProtocolSearch.entity().getProtocol(), SearchCriteria.Op.EQ); zoneProtocolSearch.and("role", zoneProtocolSearch.entity().getRole(), SearchCriteria.Op.EQ); zoneProtocolSearch.done(); + + zonesInSearch = createSearchBuilder(); + zonesInSearch.and("zonesIn", zonesInSearch.entity().getDcId(), SearchCriteria.Op.IN); + zonesInSearch.done(); + } @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -191,4 +198,12 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem List results = listBy(sc, filter); return results.size() == 0 ? null : results.get(0); } + + + @Override + public List listImageStoresByZoneIds(Long... zoneIds) { + SearchCriteria sc = zonesInSearch.create(); + sc.setParametersIfNotNull("zonesIn", zoneIds); + return listBy(sc); + } } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 97da7e83e7b..a8c239b8f6b 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -276,6 +276,8 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index b36814528c0..27170fcac14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -44,6 +44,21 @@ CREATE TABLE IF NOT EXISTS `cloud`.`quarantined_ips` ( -- create_public_parameter_on_roles. #6960 ALTER TABLE `cloud`.`roles` ADD COLUMN `public_role` tinyint(1) NOT NULL DEFAULT '1' COMMENT 'Indicates whether the role will be visible to all users (public) or only to root admins (private). If this parameter is not specified during the creation of the role its value will be defaulted to true (public).'; +-- Create heuristic table for dynamic allocating resources to the secondary storage +CREATE TABLE IF NOT EXISTS `cloud`.`heuristics` ( + `id` bigint(20) unsigned NOT NULL auto_increment, + `uuid` varchar(255) UNIQUE NOT NULL, + `name` text NOT NULL, + `description` text DEFAULT NULL, + `zone_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the zone to apply the heuristic, foreign key to `data_center` table', + `type` varchar(255) NOT NULL, + `heuristic_rule` text NOT NULL COMMENT 'JS script that defines to which secondary storage the resource will be allocated.', + `created` datetime NOT NULL, + `removed` datetime DEFAULT NULL, + PRIMARY KEY (`id`), + CONSTRAINT `fk_heuristics__zone_id` FOREIGN KEY(`zone_id`) REFERENCES `cloud`.`data_center`(`id`) +); + -- Add tables for VM Scheduler DROP TABLE IF EXISTS `cloud`.`vm_schedule`; CREATE TABLE `cloud`.`vm_schedule` ( diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 6d25c481537..5bb0d19be74 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -94,7 +94,7 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, @Override public ImageStoreEntity getImageStore(String uuid) { ImageStoreVO dataStore = dataStoreDao.findByUuid(uuid); - return getImageStore(dataStore.getId()); + return dataStore == null ? null : getImageStore(dataStore.getId()); } @Override @@ -248,6 +248,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager, return stores; } + @Override + public List listImageStoresFilteringByZoneIds(Long... zoneIds) { + List stores = dataStoreDao.listImageStoresByZoneIds(zoneIds); + List imageStores = new ArrayList<>(); + for (ImageStoreVO store : stores) { + imageStores.add(getImageStore(store.getId())); + } + return imageStores; + } + @Override public String getConfigComponentName() { return ImageStoreProviderManager.class.getSimpleName(); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java index 4268cf6446f..9c7ee983474 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImpl.java @@ -47,12 +47,14 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.jobs.AsyncJob; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CommandResult; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyAnswer; import org.apache.cloudstack.storage.command.QuerySnapshotZoneCopyCommand; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.log4j.Logger; @@ -99,6 +101,9 @@ public class SnapshotServiceImpl implements SnapshotService { @Inject ConfigurationDao _configDao; + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + static private class CreateSnapshotContext extends AsyncRpcContext { final SnapshotInfo snapshot; final AsyncCallFuture future; @@ -297,7 +302,7 @@ public class SnapshotServiceImpl implements SnapshotService { fullSnapshot = snapshotFullBackup; } if (fullSnapshot) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } else { SnapshotInfo parentSnapshot = snapshot.getParent(); // Note that DataStore information in parentSnapshot is for primary @@ -314,12 +319,25 @@ public class SnapshotServiceImpl implements SnapshotService { } } if (parentSnapshotOnBackupStore == null) { - return dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + return getImageStoreForSnapshot(snapshot.getDataCenterId(), snapshot); } return dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), parentSnapshotOnBackupStore.getRole()); } } + /** + * Verify if the data center has heuristic rules for allocating snapshots; if there is then returns the {@link DataStore} returned by the JS script. + * Otherwise, returns {@link DataStore}s with free capacity. + */ + protected DataStore getImageStoreForSnapshot(Long dataCenterId, SnapshotInfo snapshot) { + DataStore imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(dataCenterId, HeuristicType.SNAPSHOT, snapshot); + + if (imageStore == null) { + imageStore = dataStoreMgr.getImageStoreWithFreeCapacity(snapshot.getDataCenterId()); + } + return imageStore; + } + @Override public SnapshotInfo backupSnapshot(SnapshotInfo snapshot) { SnapshotObject snapObj = (SnapshotObject)snapshot; diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java index 917fb2d9c75..6d59b6f36e0 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/SnapshotServiceImplTest.java @@ -18,6 +18,9 @@ */ package org.apache.cloudstack.storage.snapshot; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; @@ -26,8 +29,8 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotResult; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.framework.async.AsyncCallFuture; -import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher; -import org.apache.cloudstack.storage.command.CommandResult; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -40,8 +43,6 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import com.cloud.storage.DataStoreRole; - @RunWith(MockitoJUnitRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) public class SnapshotServiceImplTest { @@ -57,20 +58,28 @@ public class SnapshotServiceImplTest { SnapshotDataFactory _snapshotFactory; @Mock - AsyncCallbackDispatcher caller; + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + SnapshotInfo snapshotMock; + + @Mock + VolumeInfo volumeInfoMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + private static final long DUMMY_ID = 1L; @Test public void testRevertSnapshotWithNoPrimaryStorageEntry() throws Exception { - SnapshotInfo snapshot = Mockito.mock(SnapshotInfo.class); - VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class); - - Mockito.when(snapshot.getId()).thenReturn(1L); - Mockito.when(snapshot.getVolumeId()).thenReturn(1L); + Mockito.when(snapshotMock.getId()).thenReturn(DUMMY_ID); + Mockito.when(snapshotMock.getVolumeId()).thenReturn(DUMMY_ID); Mockito.when(_snapshotFactory.getSnapshotOnPrimaryStore(1L)).thenReturn(null); - Mockito.when(volFactory.getVolume(1L, DataStoreRole.Primary)).thenReturn(volumeInfo); + Mockito.when(volFactory.getVolume(DUMMY_ID, DataStoreRole.Primary)).thenReturn(volumeInfoMock); PrimaryDataStore store = Mockito.mock(PrimaryDataStore.class); - Mockito.when(volumeInfo.getDataStore()).thenReturn(store); + Mockito.when(volumeInfoMock.getDataStore()).thenReturn(store); PrimaryDataStoreDriver driver = Mockito.mock(PrimaryDataStoreDriver.class); Mockito.when(store.getDriver()).thenReturn(driver); @@ -80,7 +89,29 @@ public class SnapshotServiceImplTest { Mockito.when(mock.get()).thenReturn(result); Mockito.when(result.isFailed()).thenReturn(false); })) { - Assert.assertTrue(snapshotService.revertSnapshot(snapshot)); + Assert.assertTrue(snapshotService.revertSnapshot(snapshotMock)); } } + + @Test + public void getImageStoreForSnapshotTestShouldListFreeImageStoresWithNoHeuristicRule() { + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(null); + Mockito.when(snapshotMock.getDataCenterId()).thenReturn(DUMMY_ID); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreForSnapshotTestShouldReturnImageStoreReturnedByTheHeuristicRule() { + DataStore dataStore = Mockito.mock(DataStore.class); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(SnapshotInfo.class))). + thenReturn(dataStore); + + snapshotService.getImageStoreForSnapshot(DUMMY_ID, snapshotMock); + + Mockito.verify(dataStoreManagerMock, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index 60d01119302..757623e3d04 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -84,6 +84,16 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.listImageStoresByScopeExcludingReadOnly(scope); } + @Override + public List getImageStoresByZoneIds(Long... zoneIds) { + return imageDataStoreMgr.listImageStoresFilteringByZoneIds(zoneIds); + } + + @Override + public DataStore getImageStoreByUuid(String uuid) { + return imageDataStoreMgr.getImageStore(uuid); + } + @Override public DataStore getRandomImageStore(long zoneId) { List stores = getImageStoresByScope(new ZoneScope(zoneId)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index 47e2ee38307..39f42e842c1 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -80,5 +80,8 @@ public interface ImageStoreProviderManager { List listImageStoresWithFreeCapacity(List imageStores); List orderImageStoresOnFreeCapacity(List imageStores); + + List listImageStoresFilteringByZoneIds(Long... zoneIds); + long getImageStoreZoneId(long dataStoreId); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index a6dc017074b..3259835f1df 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -144,6 +144,7 @@ import org.apache.cloudstack.api.response.RollingMaintenanceHostUpdatedResponse; import org.apache.cloudstack.api.response.RollingMaintenanceResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; import org.apache.cloudstack.api.response.SSHKeyPairResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.SecurityGroupRuleResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; @@ -200,6 +201,7 @@ import org.apache.cloudstack.network.lb.ApplicationLoadBalancerRule; import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -5105,6 +5107,16 @@ public class ApiResponseHelper implements ResponseGenerator { return response; } + @Override + public SecondaryStorageHeuristicsResponse createSecondaryStorageSelectorResponse(Heuristic heuristic) { + String zoneUuid = ApiDBUtils.findZoneById(heuristic.getZoneId()).getUuid(); + SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = new SecondaryStorageHeuristicsResponse(heuristic.getUuid(), heuristic.getName(), + heuristic.getDescription(), zoneUuid, heuristic.getType(), heuristic.getHeuristicRule(), heuristic.getCreated(), heuristic.getRemoved()); + secondaryStorageHeuristicsResponse.setResponseName("secondarystorageheuristics"); + + return secondaryStorageHeuristicsResponse; + } + @Override public IpQuarantineResponse createQuarantinedIpsResponse(PublicIpQuarantine quarantinedIp) { IpQuarantineResponse quarantinedIpsResponse = new IpQuarantineResponse(); diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index fd3e3298338..6b0144045a2 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -87,6 +87,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; import org.apache.cloudstack.api.command.admin.template.ListTemplatesCmdByAdmin; import org.apache.cloudstack.api.command.admin.user.ListUsersCmd; import org.apache.cloudstack.api.command.admin.zone.ListZonesCmdByAdmin; @@ -135,6 +136,7 @@ import org.apache.cloudstack.api.response.ResourceDetailResponse; import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.ResourceTagResponse; import org.apache.cloudstack.api.response.RouterHealthCheckResultResponse; +import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; @@ -158,6 +160,9 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; import org.apache.cloudstack.storage.datastore.db.ObjectStoreDao; import org.apache.cloudstack.storage.datastore.db.ObjectStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -509,6 +514,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private VirtualMachineManager virtualMachineManager; + @Inject private VolumeDao volumeDao; @@ -518,6 +524,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ManagementServerHostDao msHostDao; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + @Inject private NetworkDao networkDao; @@ -4799,6 +4808,36 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return responseGenerator.createHealthCheckResponse(_routerDao.findById(routerId), result); } + @Override + public ListResponse listSecondaryStorageSelectors(ListSecondaryStorageSelectorsCmd cmd) { + ListResponse response = new ListResponse<>(); + Pair, Integer> result = listSecondaryStorageSelectorsInternal(cmd.getZoneId(), cmd.getType(), cmd.isShowRemoved()); + List listOfSecondaryStorageHeuristicsResponses = new ArrayList<>(); + + for (Heuristic heuristic : result.first()) { + SecondaryStorageHeuristicsResponse secondaryStorageHeuristicsResponse = responseGenerator.createSecondaryStorageSelectorResponse(heuristic); + listOfSecondaryStorageHeuristicsResponses.add(secondaryStorageHeuristicsResponse); + } + + response.setResponses(listOfSecondaryStorageHeuristicsResponses); + return response; + } + + private Pair, Integer> listSecondaryStorageSelectorsInternal(Long zoneId, String type, boolean showRemoved) { + SearchBuilder searchBuilder = secondaryStorageHeuristicDao.createSearchBuilder(); + + searchBuilder.and("zoneId", searchBuilder.entity().getZoneId(), SearchCriteria.Op.EQ); + searchBuilder.and("type", searchBuilder.entity().getType(), SearchCriteria.Op.EQ); + + searchBuilder.done(); + + SearchCriteria searchCriteria = searchBuilder.create(); + searchCriteria.setParameters("zoneId", zoneId); + searchCriteria.setParametersIfNotNull("type", type); + + return secondaryStorageHeuristicDao.searchAndCount(searchCriteria, null, showRemoved); + } + @Override public ListResponse listQuarantinedIps(ListQuarantinedIpsCmd cmd) { ListResponse response = new ListResponse<>(); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 37cdbc8c033..9e88b6f1a1a 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -233,6 +233,10 @@ import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStorageCapabilitiesCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd; @@ -3897,6 +3901,11 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(PatchSystemVMCmd.class); cmdList.add(ListGuestVlansCmd.class); cmdList.add(AssignVolumeCmd.class); + cmdList.add(ListSecondaryStorageSelectorsCmd.class); + cmdList.add(CreateSecondaryStorageSelectorCmd.class); + cmdList.add(UpdateSecondaryStorageSelectorCmd.class); + cmdList.add(RemoveSecondaryStorageSelectorCmd.class); + // Out-of-band management APIs for admins cmdList.add(EnableOutOfBandManagementForHostCmd.class); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index adbb43c03f5..ec432c96782 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -60,6 +61,9 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingSto import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.CreateSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.RemoveSecondaryStorageSelectorCmd; +import org.apache.cloudstack.api.command.admin.storage.heuristics.UpdateSecondaryStorageSelectorCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -97,6 +101,10 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.Heuristic; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DettachCommand; import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; @@ -124,6 +132,7 @@ import org.apache.cloudstack.storage.object.ObjectStore; import org.apache.cloudstack.storage.object.ObjectStoreEntity; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -359,6 +368,9 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject private AnnotationDao annotationDao; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + @Inject protected UserVmManager userVmManager; @Inject @@ -2006,6 +2018,65 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); } + + @Override + public Heuristic createSecondaryStorageHeuristic(CreateSecondaryStorageSelectorCmd cmd) { + String name = cmd.getName(); + String description = cmd.getDescription(); + long zoneId = cmd.getZoneId(); + String heuristicRule = cmd.getHeuristicRule(); + String type = cmd.getType(); + HeuristicType formattedType = EnumUtils.getEnumIgnoreCase(HeuristicType.class, type); + + if (formattedType == null) { + throw new IllegalArgumentException(String.format("The given heuristic type [%s] is not valid for creating a new secondary storage selector." + + " The valid options are %s.", type, Arrays.asList(HeuristicType.values()))); + } + + HeuristicVO heuristic = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, formattedType); + + if (heuristic != null) { + DataCenterVO dataCenter = _dcDao.findById(zoneId); + throw new CloudRuntimeException(String.format("There is already a heuristic rule in the specified %s with the type [%s].", + dataCenter, type)); + } + + validateHeuristicRule(heuristicRule); + + HeuristicVO heuristicVO = new HeuristicVO(name, description, zoneId, formattedType.toString(), heuristicRule); + return secondaryStorageHeuristicDao.persist(heuristicVO); + } + + @Override + public Heuristic updateSecondaryStorageHeuristic(UpdateSecondaryStorageSelectorCmd cmd) { + long heuristicId = cmd.getId(); + String heuristicRule = cmd.getHeuristicRule(); + + HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId); + validateHeuristicRule(heuristicRule); + heuristicVO.setHeuristicRule(heuristicRule); + + return secondaryStorageHeuristicDao.persist(heuristicVO); + } + + @Override + public void removeSecondaryStorageHeuristic(RemoveSecondaryStorageSelectorCmd cmd) { + Long heuristicId = cmd.getId(); + HeuristicVO heuristicVO = secondaryStorageHeuristicDao.findById(heuristicId); + + if (heuristicVO != null) { + secondaryStorageHeuristicDao.remove(heuristicId); + } else { + throw new CloudRuntimeException("Unable to find an active heuristic with the specified UUID."); + } + } + + protected void validateHeuristicRule(String heuristicRule) { + if (StringUtils.isBlank(heuristicRule)) { + throw new IllegalArgumentException("Unable to create a new secondary storage selector as the given heuristic rule is blank."); + } + } + public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List childDatastoreAnswerList, long hostId) { StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId); List storageTags = _storagePoolTagsDao.findStoragePoolTags(datastoreClusterPoolId); diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index b4faf2ee479..69b5f984081 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -409,7 +409,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic String format = sanitizeFormat(cmd.getFormat()); Long diskOfferingId = cmd.getDiskOfferingId(); String imageStoreUuid = cmd.getImageStoreUuid(); - DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); validateVolume(caller, ownerId, zoneId, volumeName, url, format, diskOfferingId); @@ -419,6 +418,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic RegisterVolumePayload payload = new RegisterVolumePayload(cmd.getUrl(), cmd.getChecksum(), format); vol.addPayload(payload); + DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume); volService.registerVolume(vol, store); return volume; @@ -451,7 +451,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic String format = sanitizeFormat(cmd.getFormat()); final Long diskOfferingId = cmd.getDiskOfferingId(); String imageStoreUuid = cmd.getImageStoreUuid(); - final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId); @@ -461,6 +460,8 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, format, diskOfferingId, Volume.State.NotUploaded); + final DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId, volume); + VolumeInfo vol = volFactory.getVolume(volume.getId()); RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), format); @@ -655,6 +656,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic //url can be null incase of postupload if (url != null) { _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + volume.setSize(UriUtils.getRemoteSize(url)); } return volume; diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index d958cc550ba..b886f0868f6 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -59,9 +59,11 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.commons.collections.CollectionUtils; @@ -107,7 +109,7 @@ import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; public class HypervisorTemplateAdapter extends TemplateAdapterBase { - private final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); + protected final static Logger s_logger = Logger.getLogger(HypervisorTemplateAdapter.class); @Inject DownloadMonitor _downloadMonitor; @Inject @@ -145,6 +147,8 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { @Inject private AnnotationDao annotationDao; @Inject + private HeuristicRuleHelper heuristicRuleHelper; + @Inject VMInstanceDao _vmInstanceDao; @Override @@ -271,17 +275,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } if (!profile.isDirectDownload()) { - List zones = profile.getZoneIdList(); - - //zones is null when this template is to be registered to all zones - if (zones == null){ - createTemplateWithinZone(null, profile, template); - } - else { - for (Long zId : zones) { - createTemplateWithinZone(zId, profile, template); - } - } + createTemplateWithinZones(profile, template); } else { //KVM direct download templates bypassing Secondary Storage persistDirectDownloadTemplate(template.getId(), profile.getSize()); @@ -291,46 +285,67 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { return template; } - private void createTemplateWithinZone(Long zId, TemplateProfile profile, VMTemplateVO template) { - // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zId)); - if (imageStores == null || imageStores.size() == 0) { - throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + /** + * For each zone ID in {@link TemplateProfile#zoneIdList}, verifies if there is active heuristic rules for allocating template and returns the + * {@link DataStore} returned by the heuristic rule. If there is not an active heuristic rule, then allocate it to a random {@link DataStore}, if the ISO/template is private + * or allocate it to all {@link DataStore} in the zone, if it is public. + * @param profile + * @param template + */ + protected void createTemplateWithinZones(TemplateProfile profile, VMTemplateVO template) { + List zonesIds = profile.getZoneIdList(); + + if (zonesIds == null) { + zonesIds = _dcDao.listAllZones().stream().map(DataCenterVO::getId).collect(Collectors.toList()); } + List imageStores = getImageStoresThrowsExceptionIfNotFound(zonesIds, profile); + + for (long zoneId : zonesIds) { + DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); + + if (imageStore == null) { + standardImageStoreAllocation(imageStores, template); + } else { + validateSecondaryStorageAndCreateTemplate(List.of(imageStore), template, null); + } + } + } + + protected List getImageStoresThrowsExceptionIfNotFound(List zonesIds, TemplateProfile profile) { + List imageStores = storeMgr.getImageStoresByZoneIds(zonesIds.toArray(new Long[0])); + if (imageStores == null || imageStores.size() == 0) { + throw new CloudRuntimeException(String.format("Unable to find image store to download the template [%s].", profile.getTemplate())); + } + return imageStores; + } + + protected DataStore verifyHeuristicRulesForZone(VMTemplateVO template, Long zoneId) { + HeuristicType heuristicType; + if (ImageFormat.ISO.equals(template.getFormat())) { + heuristicType = HeuristicType.ISO; + } else { + heuristicType = HeuristicType.TEMPLATE; + } + return heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, heuristicType, template); + } + + protected void standardImageStoreAllocation(List imageStores, VMTemplateVO template) { Set zoneSet = new HashSet(); Collections.shuffle(imageStores); - // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. + validateSecondaryStorageAndCreateTemplate(imageStores, template, zoneSet); + } + + protected void validateSecondaryStorageAndCreateTemplate(List imageStores, VMTemplateVO template, Set zoneSet) { for (DataStore imageStore : imageStores) { - // skip data stores for a disabled zone Long zoneId = imageStore.getScope().getScopeId(); - if (zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId); - if (zone == null) { - s_logger.warn("Unable to find zone by id " + zoneId + ", so skip downloading template to its image store " + imageStore.getId()); - continue; - } - // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - s_logger.info("Zone " + zoneId + " is disabled. Skip downloading template to its image store " + imageStore.getId()); - continue; - } - - // Check if image store has enough capacity for template - if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { - s_logger.info("Image store doesn't have enough capacity. Skip downloading template to this image store " + imageStore.getId()); - continue; - } - // We want to download private template to one of the image store in a zone - if (isPrivateTemplate(template) && zoneSet.contains(zoneId)) { - continue; - } else { - zoneSet.add(zoneId); - } + if (!isZoneAndImageStoreAvailable(imageStore, zoneId, zoneSet, isPrivateTemplate(template))) { + continue; } + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); - CreateTemplateContext context = new CreateTemplateContext(null, tmpl); + CreateTemplateContext context = new CreateTemplateContext<>(null, tmpl); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); @@ -338,6 +353,44 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } } + protected boolean isZoneAndImageStoreAvailable(DataStore imageStore, Long zoneId, Set zoneSet, boolean isTemplatePrivate) { + if (zoneId == null) { + s_logger.warn(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", imageStore)); + return false; + } + + DataCenterVO zone = _dcDao.findById(zoneId); + if (zone == null) { + s_logger.warn(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", zoneId, imageStore.getId())); + return false; + } + + if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { + s_logger.info(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, imageStore.getId())); + return false; + } + + if (!_statsCollector.imageStoreHasEnoughCapacity(imageStore)) { + s_logger.info(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", imageStore.getId())); + return false; + } + + if (zoneSet == null) { + s_logger.info(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage of zone [%s].", zone)); + return true; + } + + if (isTemplatePrivate && zoneSet.contains(zoneId)) { + s_logger.info(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; therefore, image store [%s] will be skipped.", + zone, imageStore)); + return false; + } + + s_logger.info(String.format("Private template will be allocated in image store [%s] in zone [%s].", imageStore, zone)); + zoneSet.add(zoneId); + return true; + } + @Override public List createTemplateForPostUpload(final TemplateProfile profile) { // persist entry in vm_template, vm_template_details and template_zone_ref tables, not that entry at template_store_ref is not created here, and created in createTemplateAsync. @@ -352,80 +405,27 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new CloudRuntimeException("Unable to persist the template " + profile.getTemplate()); } - if (profile.getZoneIdList() != null && profile.getZoneIdList().size() > 1) - throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time"); + List zoneIdList = profile.getZoneIdList(); - Long zoneId = null; - if (profile.getZoneIdList() != null) - zoneId = profile.getZoneIdList().get(0); - - // find all eligible image stores for this zone scope - List imageStores = storeMgr.getImageStoresByScopeExcludingReadOnly(new ZoneScope(zoneId)); - if (imageStores == null || imageStores.size() == 0) { - throw new CloudRuntimeException("Unable to find image store to download template " + profile.getTemplate()); + if (zoneIdList == null) { + throw new CloudRuntimeException("Zone ID is null, cannot upload ISO/template."); } + if (zoneIdList.size() > 1) + throw new CloudRuntimeException("Operation is not supported for more than one zone id at a time."); + + Long zoneId = zoneIdList.get(0); + DataStore imageStore = verifyHeuristicRulesForZone(template, zoneId); List payloads = new LinkedList<>(); - Set zoneSet = new HashSet(); - Collections.shuffle(imageStores); // For private templates choose a random store. TODO - Have a better algorithm based on size, no. of objects, load etc. - for (DataStore imageStore : imageStores) { - // skip data stores for a disabled zone - Long zoneId_is = imageStore.getScope().getScopeId(); - if (zoneId != null) { - DataCenterVO zone = _dcDao.findById(zoneId_is); - if (zone == null) { - s_logger.warn("Unable to find zone by id " + zoneId_is + - ", so skip downloading template to its image store " + imageStore.getId()); - continue; - } - // Check if zone is disabled - if (Grouping.AllocationState.Disabled == zone.getAllocationState()) { - s_logger.info("Zone " + zoneId_is + - " is disabled, so skip downloading template to its image store " + imageStore.getId()); - continue; - } + if (imageStore == null) { + List imageStores = getImageStoresThrowsExceptionIfNotFound(List.of(zoneId), profile); + postUploadAllocation(imageStores, template, payloads); + } else { + postUploadAllocation(List.of(imageStore), template, payloads); - // We want to download private template to one of the image store in a zone - if (isPrivateTemplate(template) && zoneSet.contains(zoneId_is)) { - continue; - } else { - zoneSet.add(zoneId_is); - } - - } - - TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); - //imageService.createTemplateAsync(tmpl, imageStore, caller); - - // persist template_store_ref entry - DataObject templateOnStore = imageStore.create(tmpl); - // update template_store_ref and template state - - EndPoint ep = _epSelector.select(templateOnStore); - if (ep == null) { - String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); - s_logger.warn(errMsg); - throw new CloudRuntimeException(errMsg); - } - - TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl - .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), - templateOnStore.getDataStore().getRole().toString()); - //using the existing max template size configuration - payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); - - Long accountId = template.getAccountId(); - Account account = _accountDao.findById(accountId); - Domain domain = _domainDao.findById(account.getDomainId()); - - payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); - payload.setAccountId(accountId); - payload.setRemoteEndPoint(ep.getPublicAddr()); - payload.setRequiresHvm(template.requiresHvm()); - payload.setDescription(template.getDisplayText()); - payloads.add(payload); } + if(payloads.isEmpty()) { throw new CloudRuntimeException("unable to find zone or an image store with enough capacity"); } @@ -435,6 +435,52 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { }); } + /** + * If the template/ISO is marked as private, then it is allocated to a random secondary storage; otherwise, allocates to every storage pool in every zone given by the + * {@link TemplateProfile#zoneIdList}. + */ + private void postUploadAllocation(List imageStores, VMTemplateVO template, List payloads) { + Set zoneSet = new HashSet(); + Collections.shuffle(imageStores); + for (DataStore imageStore : imageStores) { + Long zoneId_is = imageStore.getScope().getScopeId(); + + if (!isZoneAndImageStoreAvailable(imageStore, zoneId_is, zoneSet, isPrivateTemplate(template))) { + continue; + } + + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); + + // persist template_store_ref entry + DataObject templateOnStore = imageStore.create(tmpl); + + // update template_store_ref and template state + EndPoint ep = _epSelector.select(templateOnStore); + if (ep == null) { + String errMsg = "There is no secondary storage VM for downloading template to image store " + imageStore.getName(); + s_logger.warn(errMsg); + throw new CloudRuntimeException(errMsg); + } + + TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl + .getChecksum(), tmpl.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), + templateOnStore.getDataStore().getRole().toString()); + //using the existing max template size configuration + payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key())); + + Long accountId = template.getAccountId(); + Account account = _accountDao.findById(accountId); + Domain domain = _domainDao.findById(account.getDomainId()); + + payload.setDefaultMaxSecondaryStorageInGB(_resourceLimitMgr.findCorrectResourceLimitForAccountAndDomain(account, domain, ResourceType.secondary_storage)); + payload.setAccountId(accountId); + payload.setRemoteEndPoint(ep.getPublicAddr()); + payload.setRequiresHvm(template.requiresHvm()); + payload.setDescription(template.getDisplayText()); + payloads.add(payload); + } + } + private boolean isPrivateTemplate(VMTemplateVO template){ // if public OR featured OR system template diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index f5d385b4e10..2ed42087020 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -86,6 +86,8 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.framework.messagebus.MessageBus; import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.snapshot.SnapshotHelper; import org.apache.cloudstack.storage.command.AttachCommand; import org.apache.cloudstack.storage.command.CommandResult; @@ -98,6 +100,7 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.template.VnfTemplateManager; import org.apache.cloudstack.storage.template.VnfTemplateUtils; @@ -311,6 +314,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject VnfTemplateManager vnfTemplateManager; + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + + @Inject + private HeuristicRuleHelper heuristicRuleHelper; + private TemplateAdapter getAdapter(HypervisorType type) { TemplateAdapter adapter = null; if (type == HypervisorType.BareMetal) { @@ -451,17 +460,21 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } @Override - public DataStore getImageStore(String storeUuid, Long zoneId) { + public DataStore getImageStore(String storeUuid, Long zoneId, VolumeVO volume) { DataStore imageStore = null; if (storeUuid != null) { imageStore = _dataStoreMgr.getDataStore(storeUuid, DataStoreRole.Image); } else { - imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); + imageStore = heuristicRuleHelper.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.VOLUME, volume); if (imageStore == null) { - throw new CloudRuntimeException("cannot find an image store for zone " + zoneId); + imageStore = _dataStoreMgr.getImageStoreWithFreeCapacity(zoneId); } } + if (imageStore == null) { + throw new CloudRuntimeException(String.format("Cannot find an image store for zone [%s].", zoneId)); + } + return imageStore; } diff --git a/server/src/main/java/com/cloud/test/TestAppender.java b/server/src/main/java/com/cloud/test/TestAppender.java index d8e97faef7c..9a6ec62efc6 100644 --- a/server/src/main/java/com/cloud/test/TestAppender.java +++ b/server/src/main/java/com/cloud/test/TestAppender.java @@ -46,6 +46,7 @@ import static org.apache.log4j.Level.ERROR; import static org.apache.log4j.Level.FATAL; import static org.apache.log4j.Level.INFO; import static org.apache.log4j.Level.OFF; +import static org.apache.log4j.Level.WARN; /** * @@ -152,6 +153,7 @@ public final class TestAppender extends AppenderSkeleton { expectedPatterns.put(FATAL, new HashSet()); expectedPatterns.put(INFO, new HashSet()); expectedPatterns.put(OFF, new HashSet()); + expectedPatterns.put(WARN, new HashSet()); } public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) { checkArgument(level != null, "addExpectedPattern requires a non-null level"); diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java new file mode 100644 index 00000000000..9dfc75e06f2 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelper.java @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics; + +import com.cloud.api.ApiDBUtils; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.storage.StorageManager; +import com.cloud.storage.StorageStats; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.heuristics.presetvariables.Account; +import org.apache.cloudstack.storage.heuristics.presetvariables.Domain; +import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables; +import org.apache.cloudstack.storage.heuristics.presetvariables.SecondaryStorage; +import org.apache.cloudstack.storage.heuristics.presetvariables.Snapshot; +import org.apache.cloudstack.storage.heuristics.presetvariables.Template; +import org.apache.cloudstack.storage.heuristics.presetvariables.Volume; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; +import org.apache.log4j.Logger; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Helper class for building and injecting the heuristics preset variables into the JS script. + */ +public class HeuristicRuleHelper { + + protected static final Logger LOGGER = Logger.getLogger(HeuristicRuleHelper.class); + + private static final Long HEURISTICS_SCRIPT_TIMEOUT = StorageManager.HEURISTICS_SCRIPT_TIMEOUT.value(); + + @Inject + private DataStoreManager dataStoreManager; + + @Inject + private ImageStoreDao imageStoreDao; + + @Inject + private SecondaryStorageHeuristicDao secondaryStorageHeuristicDao; + + @Inject + private DomainDao domainDao; + + @Inject + private AccountDao accountDao; + + /** + * Returns the {@link DataStore} object if the zone, specified by the ID, has an active heuristic rule for the given {@link HeuristicType}. + * It returns null otherwise. + * @param zoneId used to search for the heuristic rules. + * @param heuristicType used for checking if there is a heuristic rule for the given {@link HeuristicType}. + * @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}. + * They are used to retrieve attributes for injecting in the JS rule. + * @return the corresponding {@link DataStore} if there is a heuristic rule, returns null otherwise. + */ + public DataStore getImageStoreIfThereIsHeuristicRule(Long zoneId, HeuristicType heuristicType, Object obj) { + HeuristicVO heuristicsVO = secondaryStorageHeuristicDao.findByZoneIdAndType(zoneId, heuristicType); + + if (heuristicsVO == null) { + LOGGER.debug(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", zoneId, heuristicType)); + return null; + } else { + LOGGER.debug(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicsVO, zoneId)); + return interpretHeuristicRule(heuristicsVO.getHeuristicRule(), heuristicType, obj, zoneId); + } + } + + /** + * Build the preset variables ({@link Template}, {@link Snapshot} and {@link Volume}) for the JS script. + */ + protected void buildPresetVariables(JsInterpreter jsInterpreter, HeuristicType heuristicType, long zoneId, Object obj) { + PresetVariables presetVariables = new PresetVariables(); + Long accountId = null; + + switch (heuristicType) { + case TEMPLATE: + case ISO: + presetVariables.setTemplate(setTemplatePresetVariable((VMTemplateVO) obj)); + accountId = ((VMTemplateVO) obj).getAccountId(); + break; + case SNAPSHOT: + presetVariables.setSnapshot(setSnapshotPresetVariable((SnapshotInfo) obj)); + accountId = ((SnapshotInfo) obj).getAccountId(); + break; + case VOLUME: + presetVariables.setVolume(setVolumePresetVariable((VolumeVO) obj)); + accountId = ((VolumeVO) obj).getAccountId(); + break; + } + presetVariables.setAccount(setAccountPresetVariable(accountId)); + presetVariables.setSecondaryStorages(setSecondaryStoragesVariable(zoneId)); + + injectPresetVariables(jsInterpreter, presetVariables); + } + + /** + * Inject the {@link PresetVariables} ({@link Template}, {@link Snapshot} and {@link Volume}) into the JS {@link JsInterpreter}. + * For each type, they can be accessed using the following variables in the script: + *
    + *
  • ISO/Template: using the variable iso or template, e.g. iso.format
    • + *
  • Snapshot: using the variable snapshot, e.g. snapshot.size
    • + *
  • Volume: using the variable volume, e.g. volume.format
    • + *
+ * @param jsInterpreter the JS script + * @param presetVariables used for injecting in the JS interpreter. + */ + protected void injectPresetVariables(JsInterpreter jsInterpreter, PresetVariables presetVariables) { + jsInterpreter.injectVariable("secondaryStorages", presetVariables.getSecondaryStorages().toString()); + + if (presetVariables.getTemplate() != null) { + jsInterpreter.injectVariable("template", presetVariables.getTemplate().toString()); + jsInterpreter.injectVariable("iso", presetVariables.getTemplate().toString()); + } + + if (presetVariables.getSnapshot() != null) { + jsInterpreter.injectVariable("snapshot", presetVariables.getSnapshot().toString()); + } + + if (presetVariables.getVolume() != null) { + jsInterpreter.injectVariable("volume", presetVariables.getVolume().toString()); + } + + if (presetVariables.getAccount() != null) { + jsInterpreter.injectVariable("account", presetVariables.getAccount().toString()); + } + } + + protected List setSecondaryStoragesVariable(long zoneId) { + List secondaryStorageList = new ArrayList<>(); + List imageStoreVOS = imageStoreDao.listStoresByZoneId(zoneId); + + for (ImageStoreVO imageStore : imageStoreVOS) { + SecondaryStorage secondaryStorage = new SecondaryStorage(); + + secondaryStorage.setName(imageStore.getName()); + secondaryStorage.setId(imageStore.getUuid()); + secondaryStorage.setProtocol(imageStore.getProtocol()); + StorageStats storageStats = ApiDBUtils.getSecondaryStorageStatistics(imageStore.getId()); + + if (storageStats != null) { + secondaryStorage.setUsedDiskSize(storageStats.getByteUsed()); + secondaryStorage.setTotalDiskSize(storageStats.getCapacityBytes()); + } + + secondaryStorageList.add(secondaryStorage); + } + return secondaryStorageList; + } + + protected Template setTemplatePresetVariable(VMTemplateVO templateVO) { + Template template = new Template(); + + template.setName(templateVO.getName()); + template.setFormat(templateVO.getFormat()); + template.setHypervisorType(templateVO.getHypervisorType()); + + return template; + } + + protected Volume setVolumePresetVariable(VolumeVO volumeVO) { + Volume volume = new Volume(); + + volume.setName(volumeVO.getName()); + volume.setFormat(volumeVO.getFormat()); + volume.setSize(volumeVO.getSize()); + + return volume; + } + + protected Snapshot setSnapshotPresetVariable(SnapshotInfo snapshotInfo) { + Snapshot snapshot = new Snapshot(); + + snapshot.setName(snapshotInfo.getName()); + snapshot.setSize(snapshotInfo.getSize()); + snapshot.setHypervisorType(snapshotInfo.getHypervisorType()); + + return snapshot; + } + + protected Account setAccountPresetVariable(Long accountId) { + if (accountId == null) { + return null; + } + + AccountVO account = accountDao.findById(accountId); + if (account == null) { + return null; + } + + Account accountVariable = new Account(); + accountVariable.setName(account.getName()); + accountVariable.setId(account.getUuid()); + + accountVariable.setDomain(setDomainPresetVariable(account.getDomainId())); + + return accountVariable; + } + + protected Domain setDomainPresetVariable(long domainId) { + DomainVO domain = domainDao.findById(domainId); + if (domain == null) { + return null; + } + Domain domainVariable = new Domain(); + domainVariable.setName(domain.getName()); + domainVariable.setId(domain.getUuid()); + + return domainVariable; + } + + /** + * This method calls {@link HeuristicRuleHelper#buildPresetVariables(JsInterpreter, HeuristicType, long, Object)} for building the preset variables and + * execute the JS script specified in the rule ({@link String}) parameter. The script is pre-injected with the preset variables, to allow the JS script to reference them + * in the code scope. + *
+ *
+ * The JS script needs to return a valid UUID ({@link String}) of a secondary storage, otherwise a {@link CloudRuntimeException} is thrown. + * @param rule the {@link String} representing the JS script. + * @param heuristicType used for building the preset variables accordingly to the {@link HeuristicType} specified. + * @param obj can be from the following classes: {@link VMTemplateVO}, {@link SnapshotInfo} and {@link VolumeVO}. + * They are used to retrieve attributes for injecting in the JS rule. + * @param zoneId used for injecting the {@link SecondaryStorage} preset variables. + * @return the {@link DataStore} returned by the script. + */ + public DataStore interpretHeuristicRule(String rule, HeuristicType heuristicType, Object obj, long zoneId) { + try (JsInterpreter jsInterpreter = new JsInterpreter(HEURISTICS_SCRIPT_TIMEOUT)) { + buildPresetVariables(jsInterpreter, heuristicType, zoneId, obj); + Object scriptReturn = jsInterpreter.executeScript(rule); + + if (!(scriptReturn instanceof String)) { + throw new CloudRuntimeException(String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", rule)); + } + + DataStore dataStore = dataStoreManager.getImageStoreByUuid((String) scriptReturn); + + if (dataStore == null) { + throw new CloudRuntimeException(String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " + + "returning a valid UUID.", scriptReturn, rule)); + } + + return dataStore; + } catch (IOException ex) { + String message = String.format("Error while executing script [%s].", rule); + LOGGER.error(message, ex); + throw new CloudRuntimeException(message, ex); + } + } + +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java new file mode 100644 index 00000000000..67750e8ec5c --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Account.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class Account extends GenericHeuristicPresetVariable { + private String id; + + private Domain domain; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } + + public Domain getDomain() { + return domain; + } + + public void setDomain(Domain domain) { + this.domain = domain; + fieldNamesToIncludeInToString.add("domain"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java new file mode 100644 index 00000000000..6565c06bfbb --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Domain.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class Domain extends GenericHeuristicPresetVariable{ + private String id; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java new file mode 100644 index 00000000000..f8ded3a864a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/GenericHeuristicPresetVariable.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import java.util.HashSet; +import java.util.Set; + +public class GenericHeuristicPresetVariable { + + protected transient Set fieldNamesToIncludeInToString = new HashSet<>(); + + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + fieldNamesToIncludeInToString.add("name"); + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, fieldNamesToIncludeInToString.toArray(new String[0])); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java new file mode 100644 index 00000000000..d0487495327 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/PresetVariables.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import java.util.List; + +public class PresetVariables { + + private Account account; + + private List secondaryStorages; + + private Template template; + + private Snapshot snapshot; + + private Volume volume; + + public List getSecondaryStorages() { + return secondaryStorages; + } + + public void setSecondaryStorages(List secondaryStorages) { + this.secondaryStorages = secondaryStorages; + } + + public Template getTemplate() { + return template; + } + + public void setTemplate(Template template) { + this.template = template; + } + + public Snapshot getSnapshot() { + return snapshot; + } + + public void setSnapshot(Snapshot snapshot) { + this.snapshot = snapshot; + } + + public Volume getVolume() { + return volume; + } + + public void setVolume(Volume volume) { + this.volume = volume; + } + + public Account getAccount() { + return account; + } + + public void setAccount(Account account) { + this.account = account; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java new file mode 100644 index 00000000000..ad7058d8336 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/SecondaryStorage.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +public class SecondaryStorage extends GenericHeuristicPresetVariable { + + private String id; + + private Long usedDiskSize; + + private Long totalDiskSize; + + private String protocol; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + fieldNamesToIncludeInToString.add("id"); + } + + public Long getUsedDiskSize() { + return usedDiskSize; + } + + public void setUsedDiskSize(Long usedDiskSize) { + this.usedDiskSize = usedDiskSize; + fieldNamesToIncludeInToString.add("usedDiskSize"); + } + + public Long getTotalDiskSize() { + return totalDiskSize; + } + + public void setTotalDiskSize(Long totalDiskSize) { + this.totalDiskSize = totalDiskSize; + fieldNamesToIncludeInToString.add("totalDiskSize"); + } + + public String getProtocol() { + return protocol; + } + + public void setProtocol(String protocol) { + this.protocol = protocol; + fieldNamesToIncludeInToString.add("protocol"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java new file mode 100644 index 00000000000..34acd394dbe --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Snapshot.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; + +public class Snapshot extends GenericHeuristicPresetVariable { + + private Long size; + + private Hypervisor.HypervisorType hypervisorType; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + fieldNamesToIncludeInToString.add("size"); + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java new file mode 100644 index 00000000000..297c95fad9a --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Template.java @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.Storage; + +public class Template extends GenericHeuristicPresetVariable { + + private Hypervisor.HypervisorType hypervisorType; + + private Storage.ImageFormat format; + + private Storage.TemplateType templateType; + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(Hypervisor.HypervisorType hypervisorType) { + this.hypervisorType = hypervisorType; + fieldNamesToIncludeInToString.add("hypervisorType"); + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + fieldNamesToIncludeInToString.add("format"); + } + + public Storage.TemplateType getTemplateType() { + return templateType; + } + + public void setTemplateType(Storage.TemplateType templateType) { + this.templateType = templateType; + fieldNamesToIncludeInToString.add("templateType"); + } +} diff --git a/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java new file mode 100644 index 00000000000..4e5e81b117f --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/heuristics/presetvariables/Volume.java @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics.presetvariables; + +import com.cloud.storage.Storage; + +public class Volume extends GenericHeuristicPresetVariable { + + private Long size; + + private Storage.ImageFormat format; + + public Long getSize() { + return size; + } + + public void setSize(Long size) { + this.size = size; + fieldNamesToIncludeInToString.add("size"); + } + + public Storage.ImageFormat getFormat() { + return format; + } + + public void setFormat(Storage.ImageFormat format) { + this.format = format; + fieldNamesToIncludeInToString.add("format"); + } +} diff --git a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java index 9d4d34f4ca6..8657c07b5ef 100644 --- a/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java +++ b/server/src/test/java/com/cloud/template/HypervisorTemplateAdapterTest.java @@ -18,21 +18,26 @@ package com.cloud.template; +import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; import com.cloud.event.UsageEventVO; import com.cloud.event.dao.UsageEventDao; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.org.Grouping; +import com.cloud.server.StatsCollector; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateZoneDao; +import com.cloud.test.TestAppender; import com.cloud.user.AccountVO; import com.cloud.user.ResourceLimitService; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; @@ -45,9 +50,12 @@ import org.apache.cloudstack.framework.events.Event; import org.apache.cloudstack.framework.events.EventBus; import org.apache.cloudstack.framework.events.EventBusException; import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.log4j.Level; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -58,6 +66,7 @@ import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import org.mockito.Spy; import org.mockito.invocation.InvocationOnMock; import org.mockito.junit.MockitoJUnitRunner; import org.mockito.stubbing.Answer; @@ -68,9 +77,12 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyLong; @@ -119,10 +131,17 @@ public class HypervisorTemplateAdapterTest { ConfigurationDao _configDao; @Mock - DataStoreManager storeMgr; + DataStoreManager dataStoreManagerMock; + @Mock + HeuristicRuleHelper heuristicRuleHelperMock; + + @Mock + StatsCollector statsCollectorMock; + + @Spy @InjectMocks - HypervisorTemplateAdapter _adapter; + HypervisorTemplateAdapter _adapter = new HypervisorTemplateAdapter(); //UsageEventUtils reflection abuse helpers private Map oldFields = new HashMap<>(); @@ -298,6 +317,282 @@ public class HypervisorTemplateAdapterTest { cleanupUsageUtils(); } + @Test + public void createTemplateWithinZonesTestZoneIdsNullShouldCallListAllZones() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(null); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_dcDao, Mockito.times(1)).listAllZones(); + } + + @Test + public void createTemplateWithinZonesTestZoneIdsNotNullShouldNotCallListAllZones() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_dcDao, Mockito.times(0)).listAllZones(); + } + + @Test + public void createTemplateWithinZonesTestZoneDoesNotHaveActiveHeuristicRulesShouldCallStandardImageStoreAllocation() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(null).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_adapter, Mockito.times(1)).standardImageStoreAllocation(Mockito.isNull(), Mockito.any(VMTemplateVO.class)); + } + + @Test + public void createTemplateWithinZonesTestZoneWithHeuristicRuleShouldCallValidateSecondaryStorageAndCreateTemplate() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + DataStore dataStoreMock = Mockito.mock(DataStore.class); + List zoneIds = List.of(1L); + + Mockito.when(templateProfileMock.getZoneIdList()).thenReturn(zoneIds); + Mockito.doReturn(null).when(_adapter).getImageStoresThrowsExceptionIfNotFound(Mockito.any(List.class), Mockito.any(TemplateProfile.class)); + Mockito.doReturn(dataStoreMock).when(_adapter).verifyHeuristicRulesForZone(Mockito.any(VMTemplateVO.class), Mockito.anyLong()); + Mockito.doNothing().when(_adapter).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); + + _adapter.createTemplateWithinZones(templateProfileMock, vmTemplateVOMock); + + Mockito.verify(_adapter, Mockito.times(1)).validateSecondaryStorageAndCreateTemplate(Mockito.any(List.class), Mockito.any(VMTemplateVO.class), Mockito.isNull()); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoresThrowsExceptionIfNotFoundTestNullImageStoreShouldThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(null); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoresThrowsExceptionIfNotFoundTestEmptyImageStoreShouldThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + List imageStoresList = new ArrayList<>(); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test + public void getImageStoresThrowsExceptionIfNotFoundTestNonEmptyImageStoreShouldNotThrowCloudRuntimeException() { + TemplateProfile templateProfileMock = Mockito.mock(TemplateProfile.class); + List zoneIds = List.of(1L); + DataStore dataStoreMock = Mockito.mock(DataStore.class); + List imageStoresList = List.of(dataStoreMock); + + Mockito.when(dataStoreManagerMock.getImageStoresByZoneIds(Mockito.anyLong())).thenReturn(imageStoresList); + + _adapter.getImageStoresThrowsExceptionIfNotFound(zoneIds, templateProfileMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateIsISOFormatShouldCheckForISOHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.ISO); + _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.ISO, vmTemplateVOMock); + } + + @Test + public void verifyHeuristicRulesForZoneTestTemplateNotISOFormatShouldCheckForTemplateHeuristicType() { + VMTemplateVO vmTemplateVOMock = Mockito.mock(VMTemplateVO.class); + + Mockito.when(vmTemplateVOMock.getFormat()).thenReturn(ImageFormat.QCOW2); + _adapter.verifyHeuristicRulesForZone(vmTemplateVOMock, 1L); + + Mockito.verify(heuristicRuleHelperMock, Mockito.times(1)).getImageStoreIfThereIsHeuristicRule(1L, HeuristicType.TEMPLATE, vmTemplateVOMock); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIdIsNullShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = null; + Set zoneSet = null; + boolean isTemplatePrivate = false; + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Zone ID is null, cannot allocate ISO/template in image store [%s].", dataStoreMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIsNullShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = null; + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.WARN, Pattern.quote(String.format("Unable to find zone by id [%s], so skip downloading template to its image store [%s].", + zoneId, dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestZoneIsDisabledShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Disabled); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone [%s] is disabled. Skip downloading template to its image store [%s].", zoneId, dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestImageStoreDoesNotHaveEnoughCapacityShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(dataStoreMock.getId()).thenReturn(2L); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(false); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Image store doesn't have enough capacity. Skip downloading template to this image store [%s].", + dataStoreMock.getId()))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestImageStoreHasEnoughCapacityAndZoneSetIsNullShouldReturnTrue() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = null; + boolean isTemplatePrivate = false; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Zone set is null; therefore, the ISO/template should be allocated in every secondary storage " + + "of zone [%s].", dataCenterVOMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertTrue(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsAlreadyAllocatedToTheSameZoneShouldReturnFalse() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = Set.of(1L); + boolean isTemplatePrivate = true; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("The template is private and it is already allocated in a secondary storage in zone [%s]; " + + "therefore, image store [%s] will be skipped.", dataCenterVOMock, dataStoreMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertFalse(result); + } + + @Test + public void isZoneAndImageStoreAvailableTestTemplateIsPrivateAndItIsNotAlreadyAllocatedToTheSameZoneShouldReturnTrue() { + DataStore dataStoreMock = Mockito.mock(DataStore.class); + Long zoneId = 1L; + Set zoneSet = new HashSet<>(); + boolean isTemplatePrivate = true; + DataCenterVO dataCenterVOMock = Mockito.mock(DataCenterVO.class); + + Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(dataCenterVOMock); + Mockito.when(dataCenterVOMock.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + Mockito.when(statsCollectorMock.imageStoreHasEnoughCapacity(any(DataStore.class))).thenReturn(true); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.INFO, Pattern.quote(String.format("Private template will be allocated in image store [%s] in zone [%s].", + dataStoreMock, dataCenterVOMock))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HypervisorTemplateAdapter.s_logger, testLogAppender); + + boolean result = _adapter.isZoneAndImageStoreAvailable(dataStoreMock, zoneId, zoneSet, isTemplatePrivate); + + testLogAppender.assertMessagesLogged(); + Assert.assertTrue(result); + } + @Test public void testCheckZoneImageStoresDirectDownloadTemplate() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); @@ -309,7 +604,7 @@ public class HypervisorTemplateAdapterTest { public void testCheckZoneImageStoresRegularTemplateWithStore() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); Mockito.when(templateVO.isDirectDownload()).thenReturn(false); - Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); + Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(List.of(Mockito.mock(DataStore.class))); _adapter.checkZoneImageStores(templateVO, List.of(1L)); } @@ -317,7 +612,7 @@ public class HypervisorTemplateAdapterTest { public void testCheckZoneImageStoresRegularTemplateNoStore() { VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); Mockito.when(templateVO.isDirectDownload()).thenReturn(false); - Mockito.when(storeMgr.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>()); + Mockito.when(dataStoreManagerMock.getImageStoresByScope(Mockito.any())).thenReturn(new ArrayList<>()); _adapter.checkZoneImageStores(templateVO, List.of(1L)); } } diff --git a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java index cb4d701d8a5..a69795cf2c8 100755 --- a/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java +++ b/server/src/test/java/com/cloud/template/TemplateManagerImplTest.java @@ -19,80 +19,6 @@ package com.cloud.template; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.anyBoolean; -import static org.mockito.Matchers.anyLong; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import javax.inject.Inject; - -import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; -import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; -import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; -import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; -import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; -import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; -import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; -import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.messagebus.MessageBus; -import org.apache.cloudstack.snapshot.SnapshotHelper; -import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; -import org.apache.cloudstack.storage.template.VnfTemplateManager; -import org.apache.cloudstack.test.utils.SpringUtils; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.FilterType; -import org.springframework.core.type.classreading.MetadataReader; -import org.springframework.core.type.classreading.MetadataReaderFactory; -import org.springframework.core.type.filter.TypeFilter; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.test.context.support.AnnotationConfigContextLoader; - import com.cloud.agent.AgentManager; import com.cloud.api.query.dao.UserVmJoinDao; import com.cloud.configuration.Resource; @@ -106,6 +32,7 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.projects.ProjectManager; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.GuestOSVO; import com.cloud.storage.Snapshot; import com.cloud.storage.SnapshotVO; @@ -117,6 +44,7 @@ import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.SnapshotDao; @@ -140,6 +68,82 @@ import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.command.user.template.CreateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; +import org.apache.cloudstack.api.command.user.template.RegisterVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; +import org.apache.cloudstack.api.command.user.template.UpdateVnfTemplateCmd; +import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageStrategyFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; +import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.messagebus.MessageBus; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.snapshot.SnapshotHelper; +import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.heuristics.HeuristicRuleHelper; +import org.apache.cloudstack.storage.template.VnfTemplateManager; +import org.apache.cloudstack.test.utils.SpringUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.type.classreading.MetadataReader; +import org.springframework.core.type.classreading.MetadataReaderFactory; +import org.springframework.core.type.filter.TypeFilter; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @RunWith(SpringJUnit4ClassRunner.class) @ContextConfiguration(loader = AnnotationConfigContextLoader.class) @@ -201,6 +205,9 @@ public class TemplateManagerImplTest { @Inject VnfTemplateManager vnfTemplateManager; + @Inject + HeuristicRuleHelper heuristicRuleHelperMock; + public class CustomThreadPoolExecutor extends ThreadPoolExecutor { AtomicInteger ai = new AtomicInteger(0); public CustomThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, @@ -596,6 +603,50 @@ public class TemplateManagerImplTest { Assert.assertEquals(template, resultTemplate); } + @Test + public void getImageStoreTestStoreUuidIsNotNullShouldReturnAValidImageStoreIfValidUuid() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(dataStore); + + templateManager.getImageStore("UUID", 1L, volumeVO); + } + + @Test(expected = CloudRuntimeException.class) + public void getImageStoreTestStoreUuidIsNotNullShouldThrowCloudRuntimeExceptionIfInvalidUuid() { + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + + templateManager.getImageStore("UUID", 1L, volumeVO); + } + + @Test + public void getImageStoreTestStoreUuidIsNullAndThereIsNoActiveHeuristicRulesShouldCallGetImageStoreWithFreeCapacity() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(null); + Mockito.when(dataStoreManager.getImageStoreWithFreeCapacity(Mockito.anyLong())).thenReturn(dataStore); + + templateManager.getImageStore(null, 1L, volumeVO); + Mockito.verify(dataStoreManager, Mockito.times(1)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + + @Test + public void getImageStoreTestStoreUuidIsNullAndThereIsActiveHeuristicRulesShouldNotCallGetImageStoreWithFreeCapacity() { + DataStore dataStore = Mockito.mock(DataStore.class); + VolumeVO volumeVO = Mockito.mock(VolumeVO.class); + + Mockito.when(dataStoreManager.getDataStore(Mockito.anyString(), Mockito.any(DataStoreRole.class))).thenReturn(null); + Mockito.when(heuristicRuleHelperMock.getImageStoreIfThereIsHeuristicRule(Mockito.anyLong(), Mockito.any(HeuristicType.class), Mockito.any(VolumeVO.class))).thenReturn(dataStore); + + templateManager.getImageStore(null, 1L, volumeVO); + Mockito.verify(dataStoreManager, Mockito.times(0)).getImageStoreWithFreeCapacity(Mockito.anyLong()); + } + @Test public void testRegisterTemplateWithTemplateType() { RegisterTemplateCmd cmd = Mockito.mock(RegisterTemplateCmd.class); @@ -915,6 +966,16 @@ public class TemplateManagerImplTest { return Mockito.mock(SnapshotService.class); } + @Bean + public SecondaryStorageHeuristicDao secondaryStorageHeuristicDao() { + return Mockito.mock(SecondaryStorageHeuristicDao.class); + } + + @Bean + public HeuristicRuleHelper heuristicRuleHelper() { + return Mockito.mock(HeuristicRuleHelper.class); + } + public static class Library implements TypeFilter { @Override public boolean match(MetadataReader mdr, MetadataReaderFactory arg1) throws IOException { diff --git a/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java b/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java new file mode 100644 index 00000000000..6bf7eef2844 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/heuristics/HeuristicRuleHelperTest.java @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.heuristics; + +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.test.TestAppender; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.secstorage.HeuristicVO; +import org.apache.cloudstack.secstorage.dao.SecondaryStorageHeuristicDao; +import org.apache.cloudstack.secstorage.heuristics.HeuristicType; +import org.apache.cloudstack.storage.heuristics.presetvariables.PresetVariables; +import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; +import org.apache.log4j.Level; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.regex.Pattern; + +@RunWith(MockitoJUnitRunner.class) +public class HeuristicRuleHelperTest { + + @Mock + SecondaryStorageHeuristicDao secondaryStorageHeuristicDaoMock; + + @Mock + HeuristicVO heuristicVOMock; + + @Mock + VMTemplateVO vmTemplateVOMock; + + @Mock + SnapshotInfo snapshotInfoMock; + + @Mock + VolumeVO volumeVOMock; + + @Mock + DataStoreManager dataStoreManagerMock; + + @Mock + DataStore dataStoreMock; + + @Spy + @InjectMocks + HeuristicRuleHelper heuristicRuleHelperSpy = new HeuristicRuleHelper(); + + @Test + public void getImageStoreIfThereIsHeuristicRuleTestZoneDoesNotHaveHeuristicRuleShouldReturnNull() { + Long zoneId = 1L; + + Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(null); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("No heuristic rules found for zone with ID [%s] and heuristic type [%s]. Returning null.", + zoneId, HeuristicType.TEMPLATE))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender); + + DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null); + + testLogAppender.assertMessagesLogged(); + Assert.assertNull(result); + } + + @Test + public void getImageStoreIfThereIsHeuristicRuleTestZoneHasHeuristicRuleShouldCallInterpretHeuristicRule() { + Long zoneId = 1L; + + Mockito.when(secondaryStorageHeuristicDaoMock.findByZoneIdAndType(Mockito.anyLong(), Mockito.any(HeuristicType.class))).thenReturn(heuristicVOMock); + Mockito.when(heuristicVOMock.getHeuristicRule()).thenReturn("rule"); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).interpretHeuristicRule(Mockito.anyString(), Mockito.any(HeuristicType.class), Mockito.isNull(), + Mockito.anyLong()); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, Pattern.quote(String.format("Found the heuristic rule %s to apply for zone with ID [%s].", heuristicVOMock, zoneId))); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(HeuristicRuleHelper.LOGGER, testLogAppender); + + DataStore result = heuristicRuleHelperSpy.getImageStoreIfThereIsHeuristicRule(zoneId, HeuristicType.TEMPLATE, null); + + testLogAppender.assertMessagesLogged(); + Assert.assertNull(result); + } + + @Test + public void buildPresetVariablesTestWithTemplateHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.TEMPLATE, 1L, vmTemplateVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithIsoHeuristicTypeShouldSetTemplateAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.ISO, 1L, vmTemplateVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setTemplatePresetVariable(Mockito.any(VMTemplateVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetSnapshotAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.SNAPSHOT, 1L, snapshotInfoMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSnapshotPresetVariable(Mockito.any(SnapshotInfo.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void buildPresetVariablesTestWithSnapshotHeuristicTypeShouldSetVolumeAndSecondaryStorageAndAccountPresetVariables() { + Mockito.doNothing().when(heuristicRuleHelperSpy).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.doReturn(null).when(heuristicRuleHelperSpy).setAccountPresetVariable(Mockito.anyLong()); + + heuristicRuleHelperSpy.buildPresetVariables(null, HeuristicType.VOLUME, 1L, volumeVOMock); + + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setVolumePresetVariable(Mockito.any(VolumeVO.class)); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setSecondaryStoragesVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).setAccountPresetVariable(Mockito.anyLong()); + Mockito.verify(heuristicRuleHelperSpy, Mockito.times(1)).injectPresetVariables(Mockito.isNull(), Mockito.any(PresetVariables.class)); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleDoesNotReturnAStringShouldThrowCloudRuntimeException() { + String heuristicRule = "1"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + + String expectedMessage = String.format("Error while interpreting heuristic rule [%s], the rule did not return a String.", heuristicRule); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L)); + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithInvalidUuidShouldThrowCloudRuntimeException() { + String heuristicRule = "'uuid'"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + Mockito.doReturn(null).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString()); + + String expectedMessage = String.format("Unable to find a secondary storage with the UUID [%s] returned by the heuristic rule [%s]. Check if the rule is " + + "returning a valid UUID.", "uuid", heuristicRule); + CloudRuntimeException assertThrows = Assert.assertThrows(CloudRuntimeException.class, + () -> heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L)); + Assert.assertEquals(expectedMessage, assertThrows.getMessage()); + } + + @Test + public void interpretHeuristicRuleTestHeuristicRuleReturnAStringWithAValidUuidShouldReturnAImageStore() { + String heuristicRule = "'uuid'"; + + Mockito.doNothing().when(heuristicRuleHelperSpy).buildPresetVariables(Mockito.any(JsInterpreter.class), Mockito.any(HeuristicType.class), Mockito.anyLong(), + Mockito.any()); + Mockito.doReturn(dataStoreMock).when(dataStoreManagerMock).getImageStoreByUuid(Mockito.anyString()); + + DataStore result = heuristicRuleHelperSpy.interpretHeuristicRule(heuristicRule, HeuristicType.TEMPLATE, volumeVOMock, 1L); + + Assert.assertNotNull(result); + } +} From a15b706fbe7809b58a687c690f71391e0bed4117 Mon Sep 17 00:00:00 2001 From: Peinthor Rene Date: Tue, 5 Dec 2023 08:29:52 +0100 Subject: [PATCH 20/50] Linstor: Allow snapshot backup also to work on non hyperconverged setups (#8271) On no access to the storage nodes, we now create a temporary resource from the snapshot and copy that data into the secondary storage. Revert works the same, just that we now also look additionally for any Linstor agent node. Also enables now backup snapshot by default. This whole BackupSnapshot functionality was introduced in 4.19, so I would be happy if this still could be merged. --- .../LinstorPrimaryDataStoreDriverImpl.java | 87 +++++++++++++++++-- .../util/LinstorConfigurationManager.java | 4 +- 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java index c0f3cb4b459..9b493ff01b9 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java @@ -527,6 +527,16 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver } } + private ResourceDefinitionCreate createResourceDefinitionCreate(String rscName, String rscGrpName) + throws ApiException { + ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate(); + ResourceDefinition rd = new ResourceDefinition(); + rd.setName(rscName); + rd.setResourceGroupName(rscGrpName); + rdCreate.setResourceDefinition(rd); + return rdCreate; + } + private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) { final String rscGrp = getRscGrp(storagePoolVO); final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress()); @@ -539,11 +549,7 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver try { s_logger.debug("Create new resource definition: " + rscName); - ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate(); - ResourceDefinition rd = new ResourceDefinition(); - rd.setName(rscName); - rd.setResourceGroupName(rscGrp); - rdCreate.setResourceDefinition(rd); + ResourceDefinitionCreate rdCreate = createResourceDefinitionCreate(rscName, rscGrp); ApiCallRcList answers = linstorApi.resourceDefinitionCreate(rdCreate); checkLinstorAnswersThrow(answers); @@ -712,6 +718,10 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver VirtualMachineManager.ExecuteInSequence.value()); Optional optEP = getDiskfullEP(linstorApi, rscName); + if (optEP.isEmpty()) { + optEP = getLinstorEP(linstorApi, rscName); + } + if (optEP.isPresent()) { Answer answer = optEP.get().sendMessage(cmd); if (!answer.getResult()) { @@ -840,6 +850,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver callback.complete(res); } + /** + * Tries to get a Linstor cloudstack end point, that is at least diskless. + * + * @param api Linstor java api object + * @param rscName resource name to make available on node + * @return Optional RemoteHostEndPoint if one could get found. + * @throws ApiException + */ private Optional getLinstorEP(DevelopersApi api, String rscName) throws ApiException { List linstorNodeNames = LinstorUtil.getLinstorNodeNames(api); Collections.shuffle(linstorNodeNames); // do not always pick the first linstor node @@ -892,6 +910,25 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return Optional.empty(); } + private String restoreResourceFromSnapshot( + DevelopersApi api, + StoragePoolVO storagePoolVO, + String rscName, + String snapshotName, + String restoredName) throws ApiException { + final String rscGrp = getRscGrp(storagePoolVO); + ResourceDefinitionCreate rdc = createResourceDefinitionCreate(restoredName, rscGrp); + api.resourceDefinitionCreate(rdc); + + SnapshotRestore sr = new SnapshotRestore(); + sr.toResource(restoredName); + api.resourceSnapshotsRestoreVolumeDefinition(rscName, snapshotName, sr); + + api.resourceSnapshotRestore(rscName, snapshotName, sr); + + return getDeviceName(api, restoredName); + } + private Answer copyTemplate(DataObject srcData, DataObject dstData) { TemplateInfo tInfo = (TemplateInfo) dstData; final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId()); @@ -929,6 +966,39 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver return answer; } + /** + * Create a temporary resource from the snapshot to backup, so we can copy the data on a diskless agent + * @param api Linstor Developer api object + * @param pool StoragePool this resource resides on + * @param rscName rscName of the snapshotted resource + * @param snapshotInfo snapshot info of the snapshot + * @param origCmd original LinstorBackupSnapshotCommand that needs to have a patched path + * @return answer from agent operation + * @throws ApiException if any Linstor api operation fails + */ + private Answer copyFromTemporaryResource( + DevelopersApi api, StoragePoolVO pool, String rscName, SnapshotInfo snapshotInfo, CopyCommand origCmd) + throws ApiException { + Answer answer; + String restoreName = rscName + "-rst"; + String snapshotName = LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid(); + String devName = restoreResourceFromSnapshot(api, pool, rscName, snapshotName, restoreName); + + Optional optEPAny = getLinstorEP(api, restoreName); + if (optEPAny.isPresent()) { + // patch the src device path to the temporary linstor resource + SnapshotObjectTO soTO = (SnapshotObjectTO)snapshotInfo.getTO(); + soTO.setPath(devName); + origCmd.setSrcTO(soTO); + answer = optEPAny.get().sendMessage(origCmd); + } else{ + answer = new Answer(origCmd, false, "Unable to get matching Linstor endpoint."); + } + // delete the temporary resource, noop if already gone + api.resourceDefinitionDelete(restoreName); + return answer; + } + protected Answer copySnapshot(DataObject srcData, DataObject destData) { String value = _configDao.getValue(Config.BackupSnapshotWait.toString()); int _backupsnapshotwait = NumbersUtil.parseInt( @@ -956,13 +1026,14 @@ public class LinstorPrimaryDataStoreDriverImpl implements PrimaryDataStoreDriver VirtualMachineManager.ExecuteInSequence.value()); cmd.setOptions(options); - Optional optEP = getDiskfullEP( - api, LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid()); + String rscName = LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid(); + Optional optEP = getDiskfullEP(api, rscName); Answer answer; if (optEP.isPresent()) { answer = optEP.get().sendMessage(cmd); } else { - answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint."); + s_logger.debug("No diskfull endpoint found to copy image, creating diskless endpoint"); + answer = copyFromTemporaryResource(api, pool, rscName, snapshotInfo, cmd); } return answer; } catch (Exception e) { diff --git a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java index 16cc24a78d4..90ebf30f7cd 100644 --- a/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java +++ b/plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java @@ -21,8 +21,8 @@ import org.apache.cloudstack.framework.config.Configurable; public class LinstorConfigurationManager implements Configurable { - public static final ConfigKey BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "false", - "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot), only works on hyperconverged setups.", true, ConfigKey.Scope.Global, null); + public static final ConfigKey BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "true", + "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot)", true, ConfigKey.Scope.Global, null); public static final ConfigKey[] CONFIG_KEYS = new ConfigKey[] { BackupSnapshots }; From 4c91cc7c6f59b1eee44dcf1d4e1c2eab96d879a2 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 6 Dec 2023 16:29:15 +0530 Subject: [PATCH 21/50] ui: change ipaddress tab label for shared networks (#8305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In UI, shared network IP addresses are shown in a tab named Public IP addresses inside the network view. Public IP addresses have their own subsection in the UI. Network → Public IP address. Shared network IP addresses are not shown in this view. This is confusing for users and Public IP addresses tab in the network view has been renamed as IP addresses for a shared network. Signed-off-by: Abhishek Kumar --- ui/public/locales/en.json | 1 + ui/src/components/view/ResourceView.vue | 8 +++++++- ui/src/config/section/network.js | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index d1f0d84e755..69a794079b5 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -1030,6 +1030,7 @@ "label.ipv6.dns1": "IPv6 DNS1", "label.ipv6.dns2": "IPv6 DNS2", "label.ipv6.subnets": "IPv6 Subnets", +"label.ip.addresses": "IP Addresses", "label.iqn": "Target IQN", "label.is.in.progress": "is in progress", "label.is.shared": "Is shared", diff --git a/ui/src/components/view/ResourceView.vue b/ui/src/components/view/ResourceView.vue index 8ff0dcb8f55..a919005d427 100644 --- a/ui/src/components/view/ResourceView.vue +++ b/ui/src/components/view/ResourceView.vue @@ -44,7 +44,7 @@ @@ -104,6 +104,10 @@ export default { autoSelectLabel: { type: String, default: '' + }, + isKVMUnmanage: { + type: Boolean, + default: false } }, data () { diff --git a/ui/src/views/tools/ImportUnmanagedInstance.vue b/ui/src/views/tools/ImportUnmanagedInstance.vue index 86decf75d57..6433e68da57 100644 --- a/ui/src/views/tools/ImportUnmanagedInstance.vue +++ b/ui/src/views/tools/ImportUnmanagedInstance.vue @@ -19,7 +19,7 @@
- + - + @@ -120,7 +120,7 @@ :value="templateType" @change="changeTemplateType"> - + {{ $t('label.template.temporary.import') }} @@ -235,7 +235,7 @@ - + + @change="onSelectRootDisk"> {{ opt.label || opt.id }} + + +
@@ -283,17 +299,40 @@ :zoneId="cluster.zoneid" :selectionEnabled="false" :filterUnimplementedNetworks="true" - filterMatchKey="broadcasturi" :hypervisor="this.cluster.hypervisortype" + :filterMatchKey="isKVMUnmanage ? undefined : 'broadcasturi'" @select-multi-network="updateMultiNetworkOffering" />
- - - - + +
+ + + +
+
+ + + + + + {{ network.label }} + + + + +
@@ -358,6 +397,26 @@ export default { type: Object, required: true }, + host: { + type: Object, + required: true + }, + pool: { + type: Object, + required: true + }, + resource: { + type: Object, + required: true + }, + isOpen: { + type: Boolean, + required: false + }, + zoneid: { + type: String, + required: false + }, importsource: { type: String, required: false @@ -366,12 +425,24 @@ export default { type: String, required: false }, - resource: { - type: Object, - required: true + exthost: { + type: String, + required: false }, - isOpen: { - type: Boolean, + username: { + type: String, + required: false + }, + password: { + type: String, + required: false + }, + tmppath: { + type: String, + required: false + }, + diskpath: { + type: String, required: false }, selectedVmwareVcenter: { @@ -384,12 +455,14 @@ export default { options: { domains: [], projects: [], + networks: [], templates: [] }, rowCount: {}, optionsLoading: { domains: false, projects: false, + networks: false, templates: false }, domains: [], @@ -397,7 +470,7 @@ export default { selectedDomainId: null, templates: [], templateLoading: false, - templateType: 'auto', + templateType: this.defaultTemplateType(), totalComputeOfferings: 0, computeOfferings: [], computeOfferingLoading: false, @@ -426,7 +499,15 @@ export default { storagePoolsForConversion: [], selectedStorageOptionForConversion: null, selectedStoragePoolForConversion: null, - showStoragePoolsForConversion: false + showStoragePoolsForConversion: false, + selectedRootDiskColumns: [ + { + key: 'name', + dataIndex: 'name', + title: this.$t('label.rootdisk') + } + ], + selectedRootDiskSources: [] } }, beforeCreate () { @@ -461,6 +542,15 @@ export default { showicon: true } }, + networks: { + list: 'listNetworks', + isLoad: true, + field: 'networkid', + options: { + zoneid: this.zoneid, + details: 'min' + } + }, templates: { list: 'listTemplates', isLoad: true, @@ -479,6 +569,21 @@ export default { } return false }, + isDiskImport () { + if (this.importsource === 'local' || this.importsource === 'shared') { + return true + } + return false + }, + isExternalImport () { + if (this.importsource === 'external') { + return true + } + return false + }, + isKVMUnmanage () { + return this.hypervisor && this.hypervisor === 'kvm' && (this.importsource === 'unmanaged' || this.importsource === 'external') + }, domainSelectOptions () { var domains = this.options.domains.map((domain) => { return { @@ -507,6 +612,19 @@ export default { }) return projects }, + networkSelectOptions () { + var networks = this.options.networks.map((network) => { + return { + label: network.name + ' (' + network.displaytext + ')', + value: network.id + } + }) + networks.unshift({ + label: '', + value: null + }) + return networks + }, templateSelectOptions () { return this.options.templates.map((template) => { return { @@ -540,6 +658,9 @@ export default { var nic = { ...nicEntry } nic.name = nic.name || nic.id nic.displaytext = nic.name + if (this.isExternalImport && nic.vlanid === -1) { + delete nic.vlanid + } if (nic.vlanid) { nic.broadcasturi = 'vlan://' + nic.vlanid if (nic.isolatedpvlan) { @@ -592,6 +713,9 @@ export default { page: 1 }) this.fetchKvmHostsForConversion() + if (this.resource.disk.length > 1) { + this.updateSelectedRootDisk() + } }, getMeta (obj, metaKeys) { var meta = [] @@ -724,6 +848,12 @@ export default { updateMultiNetworkOffering (data) { this.nicsNetworksMapping = data }, + defaultTemplateType () { + if (this.cluster.hypervisortype === 'VMWare') { + return 'auto' + } + return 'custom' + }, changeTemplateType (e) { this.templateType = e.target.value if (this.templateType === 'auto') { @@ -834,6 +964,17 @@ export default { } ] }, + onSelectRootDisk (val) { + this.selectedRootDiskIndex = val + this.updateSelectedRootDisk() + }, + updateSelectedRootDisk () { + var rootDisk = this.resource.disk[this.selectedRootDiskIndex] + rootDisk.size = rootDisk.capacity / (1024 * 1024 * 1024) + rootDisk.name = `${rootDisk.label} (${rootDisk.size} GB)` + rootDisk.meta = this.getMeta(rootDisk, { controller: 'controller', datastorename: 'datastore', position: 'position' }) + this.selectedRootDiskSources = [rootDisk] + }, handleSubmit (e) { e.preventDefault() if (this.loading) return @@ -843,12 +984,32 @@ export default { name: this.resource.name, clusterid: this.cluster.id, displayname: values.displayname, + zoneid: this.zoneid, importsource: this.importsource, - hypervisor: this.hypervisor + hypervisor: this.hypervisor, + host: this.exthost, + hostname: values.hostname, + username: this.username, + password: this.password, + hostid: this.host.id, + storageid: this.pool.id, + diskpath: this.diskpath, + temppath: this.tmppath } var importapi = 'importUnmanagedInstance' if (this.isExternalImport || this.isDiskImport || this.selectedVmwareVcenter) { importapi = 'importVm' + if (this.isDiskImport) { + if (!values.networkid) { + this.$notification.error({ + message: this.$t('message.request.failed'), + description: this.$t('message.please.enter.valid.value') + ': ' + this.$t('label.network') + }) + return + } + params.name = values.displayname + params.networkid = values.networkid + } } if (!this.computeOffering || !this.computeOffering.id) { this.$notification.error({ @@ -892,6 +1053,16 @@ export default { }) } } + if (this.isDiskImport) { + var storageType = this.computeOffering.storagetype + if (this.importsource !== storageType) { + this.$notification.error({ + message: this.$t('message.request.failed'), + description: 'Incompatible Storage. Import Source is: ' + this.importsource + '. Storage Type in service offering is: ' + storageType + }) + return + } + } if (this.selectedVmwareVcenter) { if (this.selectedVmwareVcenter.existingvcenterid) { params.existingvcenterid = this.selectedVmwareVcenter.existingvcenterid @@ -934,6 +1105,7 @@ export default { } var nicNetworkIndex = 0 var nicIpIndex = 0 + var networkcheck = new Set() for (var nicId in this.nicsNetworksMapping) { if (!this.nicsNetworksMapping[nicId].network) { this.$notification.error({ @@ -944,6 +1116,16 @@ export default { } params['nicnetworklist[' + nicNetworkIndex + '].nic'] = nicId params['nicnetworklist[' + nicNetworkIndex + '].network'] = this.nicsNetworksMapping[nicId].network + var netId = this.nicsNetworksMapping[nicId].network + if (!networkcheck.has(netId)) { + networkcheck.add(netId) + } else { + this.$notification.error({ + message: this.$t('message.request.failed'), + description: 'Same network cannot be assigned to multiple Nics' + }) + return + } nicNetworkIndex++ if ('ipAddress' in this.nicsNetworksMapping[nicId]) { if (!this.nicsNetworksMapping[nicId].ipAddress) { @@ -1010,7 +1192,7 @@ export default { for (var field of fields) { this.updateFieldValue(field, undefined) } - this.templateType = 'auto' + this.templateType = this.defaultTemplateType() this.updateComputeOffering(undefined) this.switches = {} }, @@ -1022,33 +1204,33 @@ export default { diff --git a/ui/src/views/tools/ManageInstances.vue b/ui/src/views/tools/ManageInstances.vue index 96eba539638..fc14f684e72 100644 --- a/ui/src/views/tools/ManageInstances.vue +++ b/ui/src/views/tools/ManageInstances.vue @@ -48,126 +48,278 @@
- - - - - - - VMware - - - KVM - - - - + + + + + + + + VMware + + + KVM + + + + + + + + {{ opt.label }} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {{ $t('label.clusterid') }} + {{ $t('label.zoneid') }} + + + - + - {{ opt.label }} + + + {{ zoneitem.label }} - - - - - - - - - - - - - - - {{ zoneitem.label }} - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + {{ $t('label.import.instance') }} + + + + + + + + +
+ + {{ $t('label.fetch.instances') }} + +
- + + + +
- +