diff --git a/INSTALL.md b/INSTALL.md index 5f04f2d5599..efb42678b0a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -18,19 +18,19 @@ Install tools and dependencies used for development: # yum -y install git java-17-openjdk java-17-openjdk-devel \ mysql mysql-server mkisofs git gcc python MySQL-python openssh-clients wget -Set up Maven (3.6.0): +Set up Maven (3.9.10): - # wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz - # tar -zxvf apache-maven-3.9.9-bin.tar.gz -C /usr/local + # wget https://dlcdn.apache.org/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.tar.gz + # sudo tar -zxvf apache-maven-3.9.10-bin.tar.gz -C /usr/local # cd /usr/local - # ln -s apache-maven-3.9.9 maven + # sudo ln -s apache-maven-3.9.10 maven # echo export M2_HOME=/usr/local/maven >> ~/.bashrc # or .zshrc or .profile # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile # source ~/.bashrc -Setup up NodeJS (LTS): +Setup up Node.js 16: - # curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash - + # curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash - # sudo yum install nodejs # sudo npm install -g @vue/cli npm-check-updates @@ -104,13 +104,13 @@ To install dependencies. To build the project. - $ npm build + $ npm run build For Development Mode. $ npm start -Make sure to set CS_URL=http://localhost:8080/client on .env.local file on ui. +Make sure to set `CS_URL=http://localhost:8080` on the `.env.local` file on UI. You should be able to run the management server on http://localhost:5050 diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index fcd4234a136..b7c24e5126c 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -453,22 +453,30 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater certExecutor.schedule(new PostCertificateRenewalTask(this), 5, TimeUnit.SECONDS); } - private void scheduleHostLBCheckerTask(final long checkInterval) { + private void scheduleHostLBCheckerTask(final String lbAlgorithm, final long checkInterval) { String name = "HostLBCheckerTask"; if (hostLbCheckExecutor != null && !hostLbCheckExecutor.isShutdown()) { + logger.info("Shutting down the preferred host checker task {}", name); hostLbCheckExecutor.shutdown(); try { if (!hostLbCheckExecutor.awaitTermination(1, TimeUnit.SECONDS)) { hostLbCheckExecutor.shutdownNow(); } } catch (InterruptedException e) { - logger.debug("Forcing {} shutdown as it did not shutdown in the desired time due to: {}", + logger.debug("Forcing the preferred host checker task {} shutdown as it did not shutdown in the desired time due to: {}", name, e.getMessage()); hostLbCheckExecutor.shutdownNow(); } } if (checkInterval > 0L) { - logger.info("Scheduling preferred host task with host.lb.interval={}ms", checkInterval); + if ("shuffle".equalsIgnoreCase(lbAlgorithm)) { + logger.info("Scheduling the preferred host checker task to trigger once (to apply lb algorithm '{}') after host.lb.interval={} ms", lbAlgorithm, checkInterval); + hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name))); + hostLbCheckExecutor.schedule(new PreferredHostCheckerTask(), checkInterval, TimeUnit.MILLISECONDS); + return; + } + + logger.info("Scheduling a recurring preferred host checker task with lb algorithm '{}' and host.lb.interval={} ms", lbAlgorithm, checkInterval); hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name))); hostLbCheckExecutor.scheduleAtFixedRate(new PreferredHostCheckerTask(), checkInterval, checkInterval, TimeUnit.MILLISECONDS); @@ -928,7 +936,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater return new SetupCertificateAnswer(true); } - private void processManagementServerList(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) { + private void processManagementServerList(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final boolean triggerHostLB) { if (CollectionUtils.isNotEmpty(msList) && StringUtils.isNotEmpty(lbAlgorithm)) { try { final String newMSHosts = String.format("%s%s%s", com.cloud.utils.StringUtils.toCSVList(msList), IAgentShell.hostLbAlgorithmSeparator, lbAlgorithm); @@ -941,22 +949,24 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } } shell.setAvoidHosts(avoidMsList); - if ("shuffle".equals(lbAlgorithm)) { - scheduleHostLBCheckerTask(0); - } else { - scheduleHostLBCheckerTask(shell.getLbCheckerInterval(lbCheckInterval)); + if (triggerHostLB) { + logger.info("Triggering the preferred host checker task now"); + ScheduledExecutorService hostLbExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HostLB-Executor")); + hostLbExecutor.schedule(new PreferredHostCheckerTask(), 0, TimeUnit.MILLISECONDS); + hostLbExecutor.shutdown(); } + scheduleHostLBCheckerTask(lbAlgorithm, shell.getLbCheckerInterval(lbCheckInterval)); } private Answer setupManagementServerList(final SetupMSListCommand cmd) { - processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); + processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), cmd.getTriggerHostLb()); return new SetupMSListAnswer(true); } private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) { try { if (CollectionUtils.isNotEmpty(cmd.getMsList())) { - processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); + processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), false); } Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> { migrateAgentConnection(cmd.getAvoidMsList()); @@ -1046,7 +1056,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } verifyAgentArch(ready.getArch()); - processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval()); + processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval(), false); logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName()); } diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index 6c415ec1df1..d67ce679684 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -31,8 +31,8 @@ public class VirtualMachineTO { private String name; private BootloaderType bootloader; private VirtualMachine.State state; - Type type; - int cpus; + private Type type; + private int cpus; /** 'speed' is still here since 4.0.X/4.1.X management servers do not support @@ -43,49 +43,50 @@ public class VirtualMachineTO { So this is here for backwards compatibility with 4.0.X/4.1.X management servers and newer agents. */ - Integer speed; - Integer minSpeed; - Integer maxSpeed; + private Integer speed; + private Integer minSpeed; + private Integer maxSpeed; - long minRam; - long maxRam; - String hostName; - String arch; - String os; - String platformEmulator; - String bootArgs; - String[] bootupScripts; - boolean enableHA; - boolean limitCpuUse; - boolean enableDynamicallyScaleVm; + private long minRam; + private long maxRam; + private String hostName; + private String arch; + private String os; + private String platformEmulator; + private String bootArgs; + private String[] bootupScripts; + private boolean enableHA; + private boolean limitCpuUse; + private boolean enableDynamicallyScaleVm; @LogLevel(LogLevel.Log4jLevel.Off) - String vncPassword; - String vncAddr; - Map details; - String uuid; - String bootType; - String bootMode; - boolean enterHardwareSetup; + private String vncPassword; + private String vncAddr; + private Map details; + private Map params; + private String uuid; + private String bootType; + private String bootMode; + private boolean enterHardwareSetup; - DiskTO[] disks; - NicTO[] nics; - GPUDeviceTO gpuDevice; - Integer vcpuMaxLimit; - List vmData = null; + private DiskTO[] disks; + private NicTO[] nics; + private GPUDeviceTO gpuDevice; + private Integer vcpuMaxLimit; + private List vmData = null; - String configDriveLabel = null; - String configDriveIsoRootFolder = null; - String configDriveIsoFile = null; - NetworkElement.Location configDriveLocation = NetworkElement.Location.SECONDARY; + private String configDriveLabel = null; + private String configDriveIsoRootFolder = null; + private String configDriveIsoFile = null; + private NetworkElement.Location configDriveLocation = NetworkElement.Location.SECONDARY; - Double cpuQuotaPercentage = null; + private Double cpuQuotaPercentage = null; - Map guestOsDetails = new HashMap(); - Map extraConfig = new HashMap<>(); - Map networkIdToNetworkNameMap = new HashMap<>(); - DeployAsIsInfoTO deployAsIsInfo; - String metadataManufacturer; - String metadataProductName; + private Map guestOsDetails = new HashMap(); + private Map extraConfig = new HashMap<>(); + private Map networkIdToNetworkNameMap = new HashMap<>(); + private DeployAsIsInfoTO deployAsIsInfo; + private String metadataManufacturer; + private String metadataProductName; public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader, String os, boolean enableHA, boolean limitCpuUse, String vncPassword) { @@ -260,6 +261,10 @@ public class VirtualMachineTO { this.bootupScripts = bootupScripts; } + public void setEnableHA(boolean enableHA) { + this.enableHA = enableHA; + } + public DiskTO[] getDisks() { return disks; } @@ -435,6 +440,42 @@ public class VirtualMachineTO { this.deployAsIsInfo = deployAsIsInfo; } + public void setSpeed(Integer speed) { + this.speed = speed; + } + + public void setMinSpeed(Integer minSpeed) { + this.minSpeed = minSpeed; + } + + public void setMaxSpeed(Integer maxSpeed) { + this.maxSpeed = maxSpeed; + } + + public void setMinRam(long minRam) { + this.minRam = minRam; + } + + public void setMaxRam(long maxRam) { + this.maxRam = maxRam; + } + + public void setLimitCpuUse(boolean limitCpuUse) { + this.limitCpuUse = limitCpuUse; + } + + public Map getParams() { + return params; + } + + public void setParams(Map params) { + this.params = params; + } + + public void setExtraConfig(Map extraConfig) { + this.extraConfig = extraConfig; + } + public String getMetadataManufacturer() { return metadataManufacturer; } diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index e194085a9d9..c8694e076b8 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -289,6 +289,8 @@ public class EventTypes { //registering userdata events public static final String EVENT_REGISTER_USER_DATA = "REGISTER.USER.DATA"; + public static final String EVENT_REGISTER_CNI_CONFIG = "REGISTER.CNI.CONFIG"; + public static final String EVENT_DELETE_CNI_CONFIG = "DELETE.CNI.CONFIG"; //register for user API and secret keys public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY"; @@ -688,6 +690,9 @@ public class EventTypes { public static final String EVENT_EXTERNAL_OPENDAYLIGHT_CONFIGURE_CONTROLLER = "PHYSICAL.ODLCONTROLLER.CONFIGURE"; //Guest OS related events + public static final String EVENT_GUEST_OS_CATEGORY_ADD = "GUEST.OS.CATEGORY.ADD"; + public static final String EVENT_GUEST_OS_CATEGORY_DELETE = "GUEST.OS.CATEGORY.DELETE"; + public static final String EVENT_GUEST_OS_CATEGORY_UPDATE = "GUEST.OS.CATEGORY.UPDATE"; public static final String EVENT_GUEST_OS_ADD = "GUEST.OS.ADD"; public static final String EVENT_GUEST_OS_REMOVE = "GUEST.OS.REMOVE"; public static final String EVENT_GUEST_OS_UPDATE = "GUEST.OS.UPDATE"; @@ -796,6 +801,9 @@ public class EventTypes { // Resource Limit public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE"; + // Management Server + public static final String EVENT_MANAGEMENT_SERVER_REMOVE = "MANAGEMENT.SERVER.REMOVE"; + public static final String VM_LEASE_EXPIRED = "VM.LEASE.EXPIRED"; public static final String VM_LEASE_DISABLED = "VM.LEASE.DISABLED"; public static final String VM_LEASE_CANCELLED = "VM.LEASE.CANCELLED"; @@ -1296,6 +1304,9 @@ public class EventTypes { entityEventDetails.put(EVENT_SHAREDFS_EXPUNGE, SharedFS.class); entityEventDetails.put(EVENT_SHAREDFS_RECOVER, SharedFS.class); + // Management Server + entityEventDetails.put(EVENT_MANAGEMENT_SERVER_REMOVE, "ManagementServer"); + // VM Lease entityEventDetails.put(VM_LEASE_EXPIRED, VirtualMachine.class); entityEventDetails.put(VM_LEASE_EXPIRING, VirtualMachine.class); diff --git a/api/src/main/java/com/cloud/exception/OperationTimedoutException.java b/api/src/main/java/com/cloud/exception/OperationTimedoutException.java index fe27408eb4e..66b607100d9 100644 --- a/api/src/main/java/com/cloud/exception/OperationTimedoutException.java +++ b/api/src/main/java/com/cloud/exception/OperationTimedoutException.java @@ -40,7 +40,7 @@ public class OperationTimedoutException extends CloudException { boolean _isActive; public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) { - super("Commands " + seqId + " to Host " + agentId + " timed out after " + time); + super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs"); _agentId = agentId; _seqId = seqId; _time = time; diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 8b9aa4ed791..07a0dfce041 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -53,9 +53,12 @@ public interface Host extends StateObject, Identity, Partition, HAResour return strs; } } - public static final String HOST_UEFI_ENABLE = "host.uefi.enable"; - public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; - public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + + String HOST_UEFI_ENABLE = "host.uefi.enable"; + String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; + String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_OVFTOOL_VERSION = "host.ovftool.version"; + String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; /** * @return name of the machine. diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java similarity index 82% rename from plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java rename to api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java index 591da077aec..8b5551d6a8e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java @@ -44,6 +44,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm AutoscaleRequested, ScaleUpRequested, ScaleDownRequested, + AddNodeRequested, + RemoveNodeRequested, UpgradeRequested, OperationSucceeded, OperationFailed, @@ -59,6 +61,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"), Scaling("Transient state in which resources are either getting scaled up/down"), Upgrading("Transient state in which cluster is getting upgraded"), + Importing("Transient state in which additional nodes are added as worker nodes to a cluster"), + RemovingNodes("Transient state in which additional nodes are removed from a cluster"), Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."), Recovering("State in which Kubernetes cluster is recovering from alert state"), Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"), @@ -96,6 +100,17 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert); + s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing); + s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing); + s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.Importing, Event.OperationFailed, State.Running); + s_fsm.addTransition(State.Alert, Event.OperationSucceeded, State.Running); + + s_fsm.addTransition(State.Running, Event.RemoveNodeRequested, State.RemovingNodes); + s_fsm.addTransition(State.Alert, Event.RemoveNodeRequested, State.RemovingNodes); + s_fsm.addTransition(State.RemovingNodes, Event.OperationSucceeded, State.Running); + s_fsm.addTransition(State.RemovingNodes, Event.OperationFailed, State.Running); + s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering); s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running); s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert); @@ -142,4 +157,13 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm Long getMaxSize(); Long getSecurityGroupId(); ClusterType getClusterType(); + Long getControlNodeServiceOfferingId(); + Long getWorkerNodeServiceOfferingId(); + Long getEtcdNodeServiceOfferingId(); + Long getControlNodeTemplateId(); + Long getWorkerNodeTemplateId(); + Long getEtcdNodeTemplateId(); + Long getEtcdNodeCount(); + Long getCniConfigId(); + String getCniConfigDetails(); } diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java index 84e898528b5..37b8907b454 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java @@ -18,14 +18,23 @@ package com.cloud.kubernetes.cluster; import org.apache.cloudstack.acl.ControlledEntity; +import java.util.Map; + import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.utils.component.Adapter; public interface KubernetesServiceHelper extends Adapter { + enum KubernetesClusterNodeType { + CONTROL, WORKER, ETCD, DEFAULT + } + ControlledEntity findByUuid(String uuid); ControlledEntity findByVmId(long vmId); void checkVmCanBeDestroyed(UserVm userVm); + boolean isValidNodeType(String nodeType); + Map getServiceOfferingNodeTypeMap(Map> serviceOfferingNodeTypeMap); + Map getTemplateNodeTypeMap(Map> templateNodeTypeMap); void cleanupForAccount(Account account); } diff --git a/api/src/main/java/com/cloud/network/NetworkService.java b/api/src/main/java/com/cloud/network/NetworkService.java index b8dd464b365..36d58c737cc 100644 --- a/api/src/main/java/com/cloud/network/NetworkService.java +++ b/api/src/main/java/com/cloud/network/NetworkService.java @@ -268,4 +268,6 @@ public interface NetworkService { InternalLoadBalancerElementService getInternalLoadBalancerElementByNetworkServiceProviderId(long networkProviderId); InternalLoadBalancerElementService getInternalLoadBalancerElementById(long providerId); List getInternalLoadBalancerElements(); + + boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException; } diff --git a/api/src/main/java/com/cloud/network/vpc/Vpc.java b/api/src/main/java/com/cloud/network/vpc/Vpc.java index e9a831c9d83..b94089d2d43 100644 --- a/api/src/main/java/com/cloud/network/vpc/Vpc.java +++ b/api/src/main/java/com/cloud/network/vpc/Vpc.java @@ -105,4 +105,6 @@ public interface Vpc extends ControlledEntity, Identity, InternalIdentity { String getIp6Dns1(); String getIp6Dns2(); + + boolean useRouterIpAsResolver(); } diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java index af2a9847a62..9b7a83c29c1 100644 --- a/api/src/main/java/com/cloud/network/vpc/VpcService.java +++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java @@ -48,17 +48,17 @@ public interface VpcService { * @param vpcName * @param displayText * @param cidr - * @param networkDomain TODO + * @param networkDomain TODO * @param ip4Dns1 * @param ip4Dns2 - * @param displayVpc TODO + * @param displayVpc TODO + * @param useVrIpResolver * @return * @throws ResourceAllocationException TODO */ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize, - Long asNumber, List bgpPeerIds) - throws ResourceAllocationException; + Long asNumber, List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException; /** * Persists VPC record in the database diff --git a/api/src/main/java/com/cloud/resource/ResourceState.java b/api/src/main/java/com/cloud/resource/ResourceState.java index 70738c7921b..e91cf820b08 100644 --- a/api/src/main/java/com/cloud/resource/ResourceState.java +++ b/api/src/main/java/com/cloud/resource/ResourceState.java @@ -76,6 +76,10 @@ public enum ResourceState { } } + public static List s_maintenanceStates = List.of(ResourceState.Maintenance, + ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance, + ResourceState.ErrorInPrepareForMaintenance); + public ResourceState getNextState(Event a) { return s_fsm.getNextState(this, a); } @@ -98,8 +102,7 @@ public enum ResourceState { } public static boolean isMaintenanceState(ResourceState state) { - return Arrays.asList(ResourceState.Maintenance, ResourceState.ErrorInMaintenance, - ResourceState.PrepareForMaintenance, ResourceState.ErrorInPrepareForMaintenance).contains(state); + return s_maintenanceStates.contains(state); } public static boolean canAttemptMaintenance(ResourceState state) { diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 18f3e901cd9..f7b2456ce5c 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -25,16 +25,20 @@ import org.apache.cloudstack.api.command.admin.cluster.ListClustersCmd; import org.apache.cloudstack.api.command.admin.config.ListCfgGroupsByCmd; import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; import org.apache.cloudstack.api.command.admin.config.UpdateHypervisorCapabilitiesCmd; +import org.apache.cloudstack.api.command.admin.guest.AddGuestOsCategoryCmd; import org.apache.cloudstack.api.command.admin.guest.AddGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.AddGuestOsMappingCmd; +import org.apache.cloudstack.api.command.admin.guest.DeleteGuestOsCategoryCmd; import org.apache.cloudstack.api.command.admin.guest.GetHypervisorGuestOsNamesCmd; import org.apache.cloudstack.api.command.admin.guest.ListGuestOsMappingCmd; import org.apache.cloudstack.api.command.admin.guest.RemoveGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.RemoveGuestOsMappingCmd; +import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsCategoryCmd; import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsMappingCmd; import org.apache.cloudstack.api.command.admin.host.ListHostsCmd; import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd; +import org.apache.cloudstack.api.command.admin.management.RemoveManagementServerCmd; import org.apache.cloudstack.api.command.admin.pod.ListPodsByCmd; import org.apache.cloudstack.api.command.admin.resource.ArchiveAlertsCmd; import org.apache.cloudstack.api.command.admin.resource.DeleteAlertsCmd; @@ -59,8 +63,10 @@ import org.apache.cloudstack.api.command.user.ssh.CreateSSHKeyPairCmd; import org.apache.cloudstack.api.command.user.ssh.DeleteSSHKeyPairCmd; import org.apache.cloudstack.api.command.user.ssh.ListSSHKeyPairsCmd; import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd; +import org.apache.cloudstack.api.command.user.userdata.DeleteCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; +import org.apache.cloudstack.api.command.user.userdata.RegisterCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd; @@ -168,6 +174,12 @@ public interface ManagementService { */ Pair, Integer> listGuestOSCategoriesByCriteria(ListGuestOsCategoriesCmd cmd); + GuestOsCategory addGuestOsCategory(AddGuestOsCategoryCmd cmd); + + GuestOsCategory updateGuestOsCategory(UpdateGuestOsCategoryCmd cmd); + + boolean deleteGuestOsCategory(DeleteGuestOsCategoryCmd cmd); + /** * Obtains a list of all guest OS mappings * @@ -360,17 +372,23 @@ public interface ManagementService { * The api command class. * @return The list of userdatas found. */ - Pair, Integer> listUserDatas(ListUserDataCmd cmd); + Pair, Integer> listUserDatas(ListUserDataCmd cmd, boolean forCks); + + /** + * Registers a cni configuration. + * + * @param cmd The api command class. + * @return A VO with the registered user data. + */ + UserData registerCniConfiguration(RegisterCniConfigurationCmd cmd); /** * Registers a userdata. * - * @param cmd - * The api command class. + * @param cmd The api command class. * @return A VO with the registered userdata. */ UserData registerUserData(RegisterUserDataCmd cmd); - /** * Deletes a userdata. * @@ -380,6 +398,14 @@ public interface ManagementService { */ boolean deleteUserData(DeleteUserDataCmd cmd); + /** + * Deletes user data. + * + * @param cmd + * The api command class. + * @return True on success. False otherwise. + */ + boolean deleteCniConfiguration(DeleteCniConfigurationCmd cmd); /** * Search registered key pairs for the logged in user. * @@ -481,4 +507,6 @@ public interface ManagementService { Pair patchSystemVM(PatchSystemVMCmd cmd); + boolean removeManagementServer(RemoveManagementServerCmd cmd); + } diff --git a/api/src/main/java/com/cloud/server/ResourceIconManager.java b/api/src/main/java/com/cloud/server/ResourceIconManager.java index e5111d9160b..d10b3eb0cd5 100644 --- a/api/src/main/java/com/cloud/server/ResourceIconManager.java +++ b/api/src/main/java/com/cloud/server/ResourceIconManager.java @@ -16,7 +16,9 @@ // under the License. package com.cloud.server; +import java.util.Collection; import java.util.List; +import java.util.Map; public interface ResourceIconManager { @@ -25,4 +27,8 @@ public interface ResourceIconManager { boolean deleteResourceIcon(List resourceIds, ResourceTag.ResourceObjectType resourceType); ResourceIcon getByResourceTypeAndUuid(ResourceTag.ResourceObjectType type, String resourceId); + + Map getByResourceTypeAndIds(ResourceTag.ResourceObjectType type, Collection resourceIds); + + Map getByResourceTypeAndUuids(ResourceTag.ResourceObjectType type, Collection resourceUuids); } diff --git a/api/src/main/java/com/cloud/server/ResourceTag.java b/api/src/main/java/com/cloud/server/ResourceTag.java index 9bbb5d43eae..b3026deceff 100644 --- a/api/src/main/java/com/cloud/server/ResourceTag.java +++ b/api/src/main/java/com/cloud/server/ResourceTag.java @@ -66,6 +66,7 @@ public interface ResourceTag extends ControlledEntity, Identity, InternalIdentit LBStickinessPolicy(false, true), LBHealthCheckPolicy(false, true), SnapshotPolicy(true, true), + GuestOsCategory(false, false, true), GuestOs(false, true), NetworkOffering(false, true), VpcOffering(true, false), diff --git a/api/src/main/java/com/cloud/storage/GuestOsCategory.java b/api/src/main/java/com/cloud/storage/GuestOsCategory.java index b46418d5c8f..e1ee4489158 100644 --- a/api/src/main/java/com/cloud/storage/GuestOsCategory.java +++ b/api/src/main/java/com/cloud/storage/GuestOsCategory.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.storage; +import java.util.Date; + import org.apache.cloudstack.api.Identity; import org.apache.cloudstack.api.InternalIdentity; @@ -27,4 +29,7 @@ public interface GuestOsCategory extends Identity, InternalIdentity { void setName(String name); + boolean isFeatured(); + + Date getCreated(); } diff --git a/api/src/main/java/com/cloud/template/TemplateApiService.java b/api/src/main/java/com/cloud/template/TemplateApiService.java index 5b494c308c3..6138f24c92b 100644 --- a/api/src/main/java/com/cloud/template/TemplateApiService.java +++ b/api/src/main/java/com/cloud/template/TemplateApiService.java @@ -58,10 +58,23 @@ public interface TemplateApiService { VirtualMachineTemplate prepareTemplate(long templateId, long zoneId, Long storageId); + /** + * Detach ISO from VM + * @param vmId id of the VM + * @param isoId id of the ISO (when passed). If it is not passed, it will get it from user_vm table + * @param extraParams forced, isVirtualRouter + * @return true when operation succeeds, false if not + */ + boolean detachIso(long vmId, Long isoId, Boolean... extraParams); - boolean detachIso(long vmId, boolean forced); - - boolean attachIso(long isoId, long vmId, boolean forced); + /** + * Attach ISO to a VM + * @param isoId id of the ISO to attach + * @param vmId id of the VM to attach the ISO to + * @param extraParams: forced, isVirtualRouter + * @return true when operation succeeds, false if not + */ + boolean attachIso(long isoId, long vmId, Boolean... extraParams); /** * Deletes a template diff --git a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java index d8872d5fe72..89953d225a0 100644 --- a/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java +++ b/api/src/main/java/com/cloud/template/VirtualMachineTemplate.java @@ -145,6 +145,8 @@ public interface VirtualMachineTemplate extends ControlledEntity, Identity, Inte boolean isDeployAsIs(); + boolean isForCks(); + Long getUserDataId(); UserData.UserDataOverridePolicy getUserDataOverridePolicy(); diff --git a/api/src/main/java/com/cloud/user/UserData.java b/api/src/main/java/com/cloud/user/UserData.java index fa0c50473c0..13a3c74f367 100644 --- a/api/src/main/java/com/cloud/user/UserData.java +++ b/api/src/main/java/com/cloud/user/UserData.java @@ -29,4 +29,5 @@ public interface UserData extends ControlledEntity, InternalIdentity, Identity { String getUserData(); String getParams(); + boolean isForCks(); } diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 72b18b70e18..5c1c2f9a2e5 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -20,6 +20,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import com.cloud.deploy.DeploymentPlan; import org.apache.cloudstack.api.BaseCmd.HTTPMethod; import org.apache.cloudstack.api.command.admin.vm.AssignVMCmd; import org.apache.cloudstack.api.command.admin.vm.RecoverVMCmd; @@ -111,7 +112,7 @@ public interface UserVmService { UserVm rebootVirtualMachine(RebootVMCmd cmd) throws InsufficientCapacityException, ResourceUnavailableException, ResourceAllocationException; - void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException; + void startVirtualMachine(UserVm vm, DeploymentPlan plan) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException; void startVirtualMachineForHA(VirtualMachine vm, Map params, DeploymentPlanner planner) throws InsufficientCapacityException, ResourceUnavailableException, diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index ef6ec27f172..14f15c9fd0f 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -89,6 +89,9 @@ public interface VmDetailConstants { String DEPLOY_AS_IS_CONFIGURATION = "configurationId"; String KEY_PAIR_NAMES = "keypairnames"; String CKS_CONTROL_NODE_LOGIN_USER = "controlNodeLoginUser"; + String CKS_NODE_TYPE = "node"; + String OFFERING = "offering"; + String TEMPLATE = "template"; // VMware to KVM VM migrations specific String VMWARE_TO_KVM_PREFIX = "vmware-to-kvm"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index f2f52cec969..4cd89c877b2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -61,6 +61,7 @@ public enum ApiCommandResourceType { AffinityGroup(org.apache.cloudstack.affinity.AffinityGroup.class), InternalLbVm(com.cloud.network.router.VirtualRouter.class), DedicatedGuestVlanRange(com.cloud.network.GuestVlan.class), + GuestOsCategory(com.cloud.storage.GuestOsCategory.class), GuestOs(com.cloud.storage.GuestOS.class), GuestOsMapping(com.cloud.storage.GuestOSHypervisor.class), Network(com.cloud.network.Network.class), @@ -84,7 +85,7 @@ public enum ApiCommandResourceType { ObjectStore(org.apache.cloudstack.storage.object.ObjectStore.class), Bucket(org.apache.cloudstack.storage.object.Bucket.class), QuotaTariff(org.apache.cloudstack.quota.QuotaTariff.class), - KubernetesCluster(null), + KubernetesCluster(com.cloud.kubernetes.cluster.KubernetesCluster.class), KubernetesSupportedVersion(null), SharedFS(org.apache.cloudstack.storage.sharedfs.SharedFS.class); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 304e43009f2..22e7e807502 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -119,6 +119,10 @@ public class ApiConstants { public static final String CN = "cn"; public static final String COMMAND = "command"; public static final String CMD_EVENT_TYPE = "cmdeventtype"; + public static final String CNI_CONFIG = "cniconfig"; + public static final String CNI_CONFIG_ID = "cniconfigurationid"; + public static final String CNI_CONFIG_DETAILS = "cniconfigdetails"; + public static final String CNI_CONFIG_NAME = "cniconfigname"; public static final String COMPONENT = "component"; public static final String CPU_CORE_PER_SOCKET = "cpucorepersocket"; public static final String CPU_NUMBER = "cpunumber"; @@ -140,6 +144,7 @@ public class ApiConstants { public static final String ENCRYPT_FORMAT = "encryptformat"; public static final String ENCRYPT_ROOT = "encryptroot"; public static final String ENCRYPTION_SUPPORTED = "encryptionsupported"; + public static final String ETCD_IPS = "etcdips"; public static final String MIN_IOPS = "miniops"; public static final String MAX_IOPS = "maxiops"; public static final String HYPERVISOR_SNAPSHOT_RESERVE = "hypervisorsnapshotreserve"; @@ -299,6 +304,7 @@ public class ApiConstants { public static final String IS_EXTRACTABLE = "isextractable"; public static final String IS_FEATURED = "isfeatured"; public static final String IS_IMPLICIT = "isimplicit"; + public static final String IS_ISO = "isiso"; public static final String IS_PORTABLE = "isportable"; public static final String IS_PUBLIC = "ispublic"; public static final String IS_PERSISTENT = "ispersistent"; @@ -332,6 +338,7 @@ public class ApiConstants { public static final String LBID = "lbruleid"; public static final String LB_PROVIDER = "lbprovider"; public static final String MAC_ADDRESS = "macaddress"; + public static final String MANUAL_UPGRADE = "manualupgrade"; public static final String MAX = "max"; public static final String MAX_SNAPS = "maxsnaps"; public static final String MAX_BACKUPS = "maxbackups"; @@ -343,6 +350,7 @@ public class ApiConstants { public static final String MIGRATIONS = "migrations"; public static final String MEMORY = "memory"; public static final String MODE = "mode"; + public static final String MOUNT_CKS_ISO_ON_VR = "mountcksisoonvr"; public static final String MULTI_ARCH = "ismultiarch"; public static final String NSX_MODE = "nsxmode"; public static final String NETWORK_MODE = "networkmode"; @@ -359,6 +367,7 @@ public class ApiConstants { public static final String NIC_PACKED_VIRTQUEUES_ENABLED = "nicpackedvirtqueuesenabled"; public static final String NEW_START_IP = "newstartip"; public static final String NEW_END_IP = "newendip"; + public static final String KUBERNETES_NODE_VERSION = "kubernetesnodeversion"; public static final String NUM_RETRIES = "numretries"; public static final String OFFER_HA = "offerha"; public static final String OS_DISTRIBUTION = "osdistribution"; @@ -432,6 +441,7 @@ public class ApiConstants { public static final String PUBLIC_END_PORT = "publicendport"; public static final String PUBLIC_ZONE = "publiczone"; public static final String PURGE_RESOURCES = "purgeresources"; + public static final String REBALANCE = "rebalance"; public static final String RECEIVED_BYTES = "receivedbytes"; public static final String RECONNECT = "reconnect"; public static final String RECOVER = "recover"; @@ -546,6 +556,7 @@ public class ApiConstants { public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; public static final String USER_SECRET_KEY = "usersecretkey"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; + public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; public static final String VALUE = "value"; public static final String VIRTUAL_MACHINE_ID = "virtualmachineid"; @@ -562,6 +573,12 @@ public class ApiConstants { public static final String VLAN = "vlan"; public static final String VLAN_RANGE = "vlanrange"; + public static final String WORKER_SERVICE_OFFERING_ID = "workerofferingid"; + public static final String WORKER_SERVICE_OFFERING_NAME = "workerofferingname"; + public static final String CONTROL_SERVICE_OFFERING_ID = "controlofferingid"; + public static final String CONTROL_SERVICE_OFFERING_NAME = "controlofferingname"; + public static final String ETCD_SERVICE_OFFERING_ID = "etcdofferingid"; + public static final String ETCD_SERVICE_OFFERING_NAME = "etcdofferingname"; public static final String REMOVE_VLAN = "removevlan"; public static final String VLAN_ID = "vlanid"; public static final String ISOLATED_PVLAN = "isolatedpvlan"; @@ -912,6 +929,7 @@ public class ApiConstants { public static final String SPLIT_CONNECTIONS = "splitconnections"; public static final String FOR_VPC = "forvpc"; public static final String FOR_NSX = "fornsx"; + public static final String FOR_CKS = "forcks"; public static final String NSX_SUPPORT_LB = "nsxsupportlb"; public static final String NSX_SUPPORTS_INTERNAL_LB = "nsxsupportsinternallb"; public static final String FOR_TUNGSTEN = "fortungsten"; @@ -1120,6 +1138,10 @@ public class ApiConstants { public static final String MASTER_NODES = "masternodes"; public static final String NODE_IDS = "nodeids"; public static final String CONTROL_NODES = "controlnodes"; + public static final String ETCD_NODES = "etcdnodes"; + public static final String EXTERNAL_NODES = "externalnodes"; + public static final String IS_EXTERNAL_NODE = "isexternalnode"; + public static final String IS_ETCD_NODE = "isetcdnode"; public static final String MIN_SEMANTIC_VERSION = "minimumsemanticversion"; public static final String MIN_KUBERNETES_VERSION_ID = "minimumkubernetesversionid"; public static final String NODE_ROOT_DISK_SIZE = "noderootdisksize"; @@ -1128,6 +1150,8 @@ public class ApiConstants { public static final String AUTOSCALING_ENABLED = "autoscalingenabled"; public static final String MIN_SIZE = "minsize"; public static final String MAX_SIZE = "maxsize"; + public static final String NODE_TYPE_OFFERING_MAP = "nodeofferings"; + public static final String NODE_TYPE_TEMPLATE_MAP = "nodetemplates"; public static final String BOOT_TYPE = "boottype"; public static final String BOOT_MODE = "bootmode"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java b/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java index 03dc37325d4..616c37484d8 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java @@ -22,6 +22,7 @@ package org.apache.cloudstack.api; */ public enum ApiErrorCode { + BAD_REQUEST(400), UNAUTHORIZED(401), UNAUTHORIZED2FA(511), METHOD_NOT_ALLOWED(405), diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java index cbbcdc3bda4..cb75939d6bc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiServerService.java @@ -48,4 +48,6 @@ public interface ApiServerService { boolean forgotPassword(UserAccount userAccount, Domain domain); boolean resetPassword(UserAccount userAccount, String token, String password); + + boolean isPostRequestsAndTimestampsEnforced(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index e2d132c2ae6..3288eb58d75 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -22,20 +22,16 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.cloud.bgp.ASNumber; -import com.cloud.bgp.ASNumberRange; - -import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.affinity.AffinityGroup; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants.HostDetails; import org.apache.cloudstack.api.ApiConstants.VMDetails; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; -import org.apache.cloudstack.api.response.AccountResponse; -import org.apache.cloudstack.api.response.ApplicationLoadBalancerResponse; import org.apache.cloudstack.api.response.ASNRangeResponse; import org.apache.cloudstack.api.response.ASNumberResponse; +import org.apache.cloudstack.api.response.AccountResponse; +import org.apache.cloudstack.api.response.ApplicationLoadBalancerResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.api.response.AutoScalePolicyResponse; import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse; @@ -60,10 +56,10 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.ExtractResponse; -import org.apache.cloudstack.api.response.SharedFSResponse; import org.apache.cloudstack.api.response.FirewallResponse; import org.apache.cloudstack.api.response.FirewallRuleResponse; import org.apache.cloudstack.api.response.GlobalLoadBalancerResponse; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.api.response.GuestOsMappingResponse; import org.apache.cloudstack.api.response.GuestVlanRangeResponse; @@ -73,11 +69,11 @@ import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.HypervisorCapabilitiesResponse; import org.apache.cloudstack.api.response.HypervisorGuestOsNamesResponse; import org.apache.cloudstack.api.response.IPAddressResponse; -import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.ImageStoreResponse; import org.apache.cloudstack.api.response.InstanceGroupResponse; import org.apache.cloudstack.api.response.InternalLoadBalancerElementResponse; import org.apache.cloudstack.api.response.IpForwardingRuleResponse; +import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.IsolationMethodResponse; import org.apache.cloudstack.api.response.LBHealthCheckResponse; import org.apache.cloudstack.api.response.LBStickinessResponse; @@ -115,6 +111,7 @@ import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; +import org.apache.cloudstack.api.response.SharedFSResponse; import org.apache.cloudstack.api.response.Site2SiteCustomerGatewayResponse; import org.apache.cloudstack.api.response.Site2SiteVpnConnectionResponse; import org.apache.cloudstack.api.response.Site2SiteVpnGatewayResponse; @@ -159,10 +156,14 @@ import org.apache.cloudstack.region.PortableIp; import org.apache.cloudstack.region.PortableIpRange; import org.apache.cloudstack.region.Region; import org.apache.cloudstack.secstorage.heuristics.Heuristic; -import org.apache.cloudstack.storage.sharedfs.SharedFS; +import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.storage.sharedfs.SharedFS; import org.apache.cloudstack.usage.Usage; +import org.apache.cloudstack.vm.UnmanagedInstanceTO; +import com.cloud.bgp.ASNumber; +import com.cloud.bgp.ASNumberRange; import com.cloud.capacity.Capacity; import com.cloud.configuration.ResourceCount; import com.cloud.configuration.ResourceLimit; @@ -223,10 +224,11 @@ import com.cloud.projects.ProjectAccount; import com.cloud.projects.ProjectInvitation; import com.cloud.region.ha.GlobalLoadBalancerRule; import com.cloud.resource.RollingMaintenanceManager; -import com.cloud.server.ResourceTag; import com.cloud.server.ResourceIcon; +import com.cloud.server.ResourceTag; import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSHypervisor; +import com.cloud.storage.GuestOsCategory; import com.cloud.storage.ImageStore; import com.cloud.storage.Snapshot; import com.cloud.storage.StoragePool; @@ -240,14 +242,13 @@ import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.user.UserData; import com.cloud.uservm.UserVm; -import com.cloud.utils.net.Ip; import com.cloud.utils.Pair; +import com.cloud.utils.net.Ip; import com.cloud.vm.InstanceGroup; import com.cloud.vm.Nic; import com.cloud.vm.NicSecondaryIp; import com.cloud.vm.VirtualMachine; import com.cloud.vm.snapshot.VMSnapshot; -import org.apache.cloudstack.vm.UnmanagedInstanceTO; public interface ResponseGenerator { UserResponse createUserResponse(UserAccount user); @@ -485,6 +486,10 @@ public interface ResponseGenerator { AutoScaleVmGroupResponse createAutoScaleVmGroupResponse(AutoScaleVmGroup vmGroup); + GuestOSCategoryResponse createGuestOSCategoryResponse(GuestOsCategory guestOsCategory); + + GuestOSCategoryResponse createGuestOSCategoryResponse(GuestOsCategory guestOsCategory, boolean showIcon); + GuestOSResponse createGuestOSResponse(GuestOS os); GuestOsMappingResponse createGuestOSMappingResponse(GuestOSHypervisor osHypervisor); @@ -572,4 +577,6 @@ public interface ResponseGenerator { BackupRepositoryResponse createBackupRepositoryResponse(BackupRepository repository); SharedFSResponse createSharedFSResponse(ResponseView view, SharedFS sharedFS); + + void updateTemplateIsoResponsesForIcons(List responses, ResourceTag.ResourceObjectType type); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCategoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCategoryCmd.java new file mode 100644 index 00000000000..f099de43f5d --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCategoryCmd.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.guest; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; + +import com.cloud.storage.GuestOsCategory; +import com.cloud.user.Account; + +@APICommand(name = "addOsCategory", + description = "Adds a new OS category", + responseObject = GuestOSCategoryResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.21.0", + authorized = {RoleType.Admin}) +public class AddGuestOsCategoryCmd extends BaseCmd { + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "Name of the OS category", + required = true) + private String name; + + @Parameter(name = ApiConstants.IS_FEATURED, type = CommandType.BOOLEAN, + description = "Whether the category is featured or not") + private Boolean featured; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getName() { + return name; + } + + public boolean isFeatured() { + return Boolean.TRUE.equals(featured); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + GuestOsCategory guestOs = _mgr.addGuestOsCategory(this); + if (guestOs != null) { + GuestOSCategoryResponse response = _responseGenerator.createGuestOSCategoryResponse(guestOs); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add new OS category"); + } + + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.GuestOsCategory; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/DeleteGuestOsCategoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/DeleteGuestOsCategoryCmd.java new file mode 100644 index 00000000000..8cf62ee2aab --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/DeleteGuestOsCategoryCmd.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.guest; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; +import org.apache.cloudstack.api.response.SuccessResponse; + +import com.cloud.user.Account; + + +@APICommand(name = "deleteOsCategory", + description = "Deletes an OS category", + responseObject = SuccessResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.21.0", + authorized = {RoleType.Admin}) +public class DeleteGuestOsCategoryCmd extends BaseCmd { + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, + required = true, description = "ID of the OS category") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + boolean result = _mgr.deleteGuestOsCategory(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to remove OS category"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.GuestOsCategory; + } + + @Override + public Long getApiResourceId() { + return getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCategoryCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCategoryCmd.java new file mode 100644 index 00000000000..4041967abe8 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCategoryCmd.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.guest; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; + +import com.cloud.storage.GuestOsCategory; +import com.cloud.user.Account; + +@APICommand(name = "updateOsCategory", + description = "Updates an OS category", + responseObject = GuestOSCategoryResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.21.0", + authorized = {RoleType.Admin}) +public class UpdateGuestOsCategoryCmd extends BaseCmd { + + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, + required = true, description = "ID of the OS category") + private Long id; + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "Name for the OS category") + private String name; + + @Parameter(name = ApiConstants.IS_FEATURED, type = CommandType.BOOLEAN, + description = "Whether the category is featured or not") + private Boolean featured; + + @Parameter(name = ApiConstants.SORT_KEY, type = CommandType.INTEGER, + description = "sort key of the OS category for listing") + private Integer sortKey; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + public Boolean isFeatured() { + return featured; + } + + public Integer getSortKey() { + return sortKey; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public void execute() { + GuestOsCategory guestOs = _mgr.updateGuestOsCategory(this); + if (guestOs != null) { + GuestOSCategoryResponse response = _responseGenerator.createGuestOSCategoryResponse(guestOs); + response.setResponseName(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update OS category"); + } + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.GuestOsCategory; + } + + @Override + public Long getApiResourceId() { + return getId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java index c98cd149ef3..59909e09854 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmd.java @@ -16,8 +16,12 @@ // under the License. package org.apache.cloudstack.api.command.admin.guest; -import org.apache.commons.collections.MapUtils; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -25,18 +29,14 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.GuestOSResponse; -import org.apache.cloudstack.acl.RoleType; +import org.apache.commons.collections.MapUtils; import com.cloud.event.EventTypes; import com.cloud.storage.GuestOS; import com.cloud.user.Account; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; - @APICommand(name = "updateGuestOs", description = "Updates the information about Guest OS", responseObject = GuestOSResponse.class, since = "4.4.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class UpdateGuestOsCmd extends BaseAsyncCmd { @@ -50,7 +50,7 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSResponse.class, required = true, description = "UUID of the Guest OS") private Long id; - @Parameter(name = ApiConstants.OS_DISPLAY_NAME, type = CommandType.STRING, required = true, description = "Unique display name for Guest OS") + @Parameter(name = ApiConstants.OS_DISPLAY_NAME, type = CommandType.STRING, description = "Unique display name for Guest OS") private String osDisplayName; @Parameter(name = ApiConstants.DETAILS, type = CommandType.MAP, required = false, description = "Map of (key/value pairs)") @@ -59,6 +59,12 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { @Parameter(name="forDisplay", type=CommandType.BOOLEAN, description="whether this guest OS is available for end users", authorized = {RoleType.Admin}) private Boolean display; + @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, + description = "the ID of the OS category", since = "4.21.0") + private Long osCategoryId; + + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -71,9 +77,9 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { return osDisplayName; } - public Map getDetails() { - Map detailsMap = new HashMap<>();; - if (MapUtils.isNotEmpty(detailsMap)) { + public Map getDetails() { + Map detailsMap = new HashMap<>(); + if (MapUtils.isNotEmpty(details)) { Collection servicesCollection = details.values(); Iterator iter = servicesCollection.iterator(); while (iter.hasNext()) { @@ -90,6 +96,10 @@ public class UpdateGuestOsCmd extends BaseAsyncCmd { return display; } + public Long getOsCategoryId() { + return osCategoryId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/management/RemoveManagementServerCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/RemoveManagementServerCmd.java new file mode 100644 index 00000000000..803b95bfa9e --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/management/RemoveManagementServerCmd.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.management; + +import com.cloud.event.EventTypes; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; + +@APICommand(name = "removeManagementServer", description = "Removes a Management Server.", responseObject = SuccessResponse.class, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = RoleType.Admin) +public class RemoveManagementServerCmd extends BaseCmd { + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, required = true, description = "the ID of the Management Server") + private Long id; + + public Long getId() { + return id; + } + + @Override + public void execute() { + boolean result = _mgr.removeManagementServer(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to remove Management Server."); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + public String getEventType() { + return EventTypes.EVENT_MANAGEMENT_SERVER_REMOVE; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java index 4f4b2631667..eafee7424ff 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/PatchSystemVMCmd.java @@ -46,7 +46,7 @@ public class PatchSystemVMCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, description = "If true, initiates copy of scripts and restart of the agent, even if the scripts version matches." + "To be used with ID parameter only") - private Boolean force; + private Boolean forced; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -58,7 +58,7 @@ public class PatchSystemVMCmd extends BaseAsyncCmd { } public boolean isForced() { - return force != null && force; + return forced != null && forced; } ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmd.java index c74514d662c..413798970bf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmd.java @@ -26,7 +26,9 @@ import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.ZoneResponse; +import com.cloud.cpu.CPU; import com.cloud.storage.GuestOsCategory; import com.cloud.utils.Pair; @@ -39,12 +41,48 @@ public class ListGuestOsCategoriesCmd extends BaseListCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, description = "list Os category by id") + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, description = "List OS category by id") private Long id; - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "list os category by name", since = "3.0.1") + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, description = "List OS category by name", since = "3.0.1") private String name; + @Parameter(name = ApiConstants.IS_FEATURED, + type = CommandType.BOOLEAN, + description = "List available OS categories by featured or not", + since = "4.21.0") + private Boolean featured; + + @Parameter(name = ApiConstants.IS_ISO, + type = CommandType.BOOLEAN, + description = "List OS categories for which an ISO is available", + since = "4.21.0") + private Boolean iso; + + @Parameter(name = ApiConstants.IS_VNF, type = CommandType.BOOLEAN, + description = "List OS categories for which a VNF template is available", + since = "4.21.0") + private Boolean vnf; + + @Parameter(name = ApiConstants.ZONE_ID, + type = CommandType.UUID, + entityType = ZoneResponse.class, + description = "List available OS categories types for the zone", + since = "4.21.0") + private Long zoneId; + + @Parameter(name = ApiConstants.ARCH, + type = CommandType.STRING, + description = "List OS categories types available for given CPU architecture", + since = "4.21.0") + private String arch; + + @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, + type = CommandType.BOOLEAN, + description = "flag to display the resource image for the OS category", + since = "4.21.0") + private Boolean showIcon; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -57,6 +95,30 @@ public class ListGuestOsCategoriesCmd extends BaseListCmd { return name; } + public Boolean isFeatured() { + return featured; + } + + public Boolean isIso() { + return iso; + } + + public Boolean isVnf() { + return vnf; + } + + public Long getZoneId() { + return zoneId; + } + + public CPU.CPUArch getArch() { + return arch == null ? null : CPU.CPUArch.fromType(arch); + } + + public boolean isShowIcon() { + return Boolean.TRUE.equals(showIcon); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -64,14 +126,11 @@ public class ListGuestOsCategoriesCmd extends BaseListCmd { @Override public void execute() { Pair, Integer> result = _mgr.listGuestOSCategoriesByCriteria(this); - ListResponse response = new ListResponse(); - List osCatResponses = new ArrayList(); + ListResponse response = new ListResponse<>(); + List osCatResponses = new ArrayList<>(); for (GuestOsCategory osCategory : result.first()) { - GuestOSCategoryResponse categoryResponse = new GuestOSCategoryResponse(); - categoryResponse.setId(osCategory.getUuid()); - categoryResponse.setName(osCategory.getName()); - - categoryResponse.setObjectName("oscategory"); + GuestOSCategoryResponse categoryResponse = _responseGenerator.createGuestOSCategoryResponse(osCategory, + isShowIcon()); osCatResponses.add(categoryResponse); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java index 292e1c6f099..78f1a4bcdee 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/DetachIsoCmd.java @@ -104,7 +104,7 @@ public class DetachIsoCmd extends BaseAsyncCmd implements UserCmd { @Override public void execute() { - boolean result = _templateService.detachIso(virtualMachineId, isForced()); + boolean result = _templateService.detachIso(virtualMachineId, null, isForced()); if (result) { UserVm userVm = _entityMgr.findById(UserVm.class, virtualMachineId); UserVmResponse response = _responseGenerator.createUserVmResponse(getResponseView(), "virtualmachine", userVm).get(0); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java index 5c4d606a93c..760a531e899 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java @@ -16,11 +16,6 @@ // under the License. package org.apache.cloudstack.api.command.user.iso; -import com.cloud.cpu.CPU; -import com.cloud.server.ResourceIcon; -import com.cloud.server.ResourceTag; -import org.apache.cloudstack.api.response.ResourceIconResponse; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -28,16 +23,17 @@ import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; - -import com.cloud.template.VirtualMachineTemplate.TemplateFilter; -import com.cloud.user.Account; import org.apache.commons.lang3.StringUtils; -import java.util.List; +import com.cloud.cpu.CPU; +import com.cloud.server.ResourceTag; +import com.cloud.template.VirtualMachineTemplate.TemplateFilter; +import com.cloud.user.Account; @APICommand(name = "listIsos", description = "Lists all available ISO files.", responseObject = TemplateResponse.class, responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -95,6 +91,11 @@ public class ListIsosCmd extends BaseListTaggedResourcesCmd implements UserCmd { since = "4.20") private String arch; + @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType= GuestOSCategoryResponse.class, + description = "the ID of the OS category for the ISO", + since = "4.21.0") + private Long osCategoryId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -173,6 +174,10 @@ public class ListIsosCmd extends BaseListTaggedResourcesCmd implements UserCmd { return CPU.CPUArch.fromType(arch); } + public Long getOsCategoryId() { + return osCategoryId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -190,24 +195,14 @@ public class ListIsosCmd extends BaseListTaggedResourcesCmd implements UserCmd { @Override public void execute() { ListResponse response = _queryService.listIsos(this); - if (response != null && response.getCount() > 0 && getShowIcon()) { - updateIsoResponse(response.getResponses()); + if (response != null && getShowIcon()) { + _responseGenerator.updateTemplateIsoResponsesForIcons(response.getResponses(), + ResourceTag.ResourceObjectType.ISO); } response.setResponseName(getCommandName()); setResponseObject(response); } - private void updateIsoResponse(List response) { - for (TemplateResponse templateResponse : response) { - ResourceIcon resourceIcon = resourceIconManager.getByResourceTypeAndUuid(ResourceTag.ResourceObjectType.ISO, templateResponse.getId()); - if (resourceIcon == null) { - continue; - } - ResourceIconResponse iconResponse = _responseGenerator.createResourceIconResponse(resourceIcon); - templateResponse.setResourceIconResponse(iconResponse); - } - } - public Long getStoragePoolId() { return null; }; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java index 8fa1a5d53eb..330224a6055 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java @@ -99,6 +99,11 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { description = "(VMware only) true if VM deployments should preserve all the configurations defined for this template", since = "4.15.1") private Boolean deployAsIs; + @Parameter(name=ApiConstants.FOR_CKS, + type = CommandType.BOOLEAN, + description = "if true, the templates would be available for deploying CKS clusters", since = "4.21.0") + protected Boolean forCks; + public String getDisplayText() { return StringUtils.isBlank(displayText) ? getName() : displayText; } @@ -168,6 +173,10 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd { Boolean.TRUE.equals(deployAsIs); } + public boolean isForCks() { + return Boolean.TRUE.equals(forCks); + } + public CPU.CPUArch getArch() { return CPU.CPUArch.fromType(arch); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java index bff65ef70a9..585114e07ad 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java @@ -16,17 +16,11 @@ // under the License. package org.apache.cloudstack.api.command.user.template; -import com.cloud.cpu.CPU; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.server.ResourceIcon; -import com.cloud.server.ResourceTag; -import org.apache.cloudstack.api.response.ResourceIconResponse; -import org.apache.commons.collections.CollectionUtils; - import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; import java.util.List; + import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -34,15 +28,20 @@ import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.user.UserCmd; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import com.cloud.cpu.CPU; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.server.ResourceTag; import com.cloud.template.VirtualMachineTemplate; import com.cloud.template.VirtualMachineTemplate.TemplateFilter; import com.cloud.user.Account; -import org.apache.commons.lang3.StringUtils; @APICommand(name = "listTemplates", description = "List all public, private, and privileged templates.", responseObject = TemplateResponse.class, entityType = {VirtualMachineTemplate.class}, responseView = ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) @@ -106,11 +105,21 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User since = "4.19.0") private Boolean isVnf; + @Parameter(name = ApiConstants.FOR_CKS, type = CommandType.BOOLEAN, + description = "list templates that can be used to deploy CKS clusters", + since = "4.21.0") + private Boolean forCks; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, description = "the CPU arch of the template. Valid options are: x86_64, aarch64", since = "4.20") private String arch; + @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType= GuestOSCategoryResponse.class, + description = "the ID of the OS category for the template", + since = "4.21.0") + private Long osCategoryId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -198,6 +207,8 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User return isVnf; } + public Boolean getForCks() { return forCks; } + public CPU.CPUArch getArch() { if (StringUtils.isBlank(arch)) { return null; @@ -205,6 +216,10 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User return CPU.CPUArch.fromType(arch); } + public Long getOsCategoryId() { + return osCategoryId; + } + @Override public String getCommandName() { return s_name; @@ -218,24 +233,14 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User @Override public void execute() { ListResponse response = _queryService.listTemplates(this); - if (response != null && response.getCount() > 0 && getShowIcon()) { - updateTemplateResponse(response.getResponses()); + if (response != null && getShowIcon()) { + _responseGenerator.updateTemplateIsoResponsesForIcons(response.getResponses(), + ResourceTag.ResourceObjectType.Template); } response.setResponseName(getCommandName()); setResponseObject(response); } - private void updateTemplateResponse(List response) { - for (TemplateResponse templateResponse : response) { - ResourceIcon resourceIcon = resourceIconManager.getByResourceTypeAndUuid(ResourceTag.ResourceObjectType.Template, templateResponse.getId()); - if (resourceIcon == null) { - continue; - } - ResourceIconResponse iconResponse = _responseGenerator.createResourceIconResponse(resourceIcon); - templateResponse.setResourceIconResponse(iconResponse); - } - } - public List getIds() { if (ids == null) { return Collections.emptyList(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java index 1f968b869b9..6ea149fd90d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/RegisterTemplateCmd.java @@ -168,6 +168,11 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { description = "(VMware only) true if VM deployments should preserve all the configurations defined for this template", since = "4.15.1") protected Boolean deployAsIs; + @Parameter(name=ApiConstants.FOR_CKS, + type = CommandType.BOOLEAN, + description = "if true, the templates would be available for deploying CKS clusters", since = "4.21.0") + protected Boolean forCks; + @Parameter(name = ApiConstants.TEMPLATE_TYPE, type = CommandType.STRING, description = "the type of the template. Valid options are: USER/VNF (for all users) and SYSTEM/ROUTING/BUILTIN (for admins only).", since = "4.19.0") @@ -295,6 +300,10 @@ public class RegisterTemplateCmd extends BaseCmd implements UserCmd { Boolean.TRUE.equals(deployAsIs); } + public boolean isForCks() { + return Boolean.TRUE.equals(forCks); + } + public String getTemplateType() { return templateType; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java index dbbd771293a..20849d1ba6c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/UpdateTemplateCmd.java @@ -46,6 +46,11 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use @Parameter(name = ApiConstants.TEMPLATE_TAG, type = CommandType.STRING, description = "the tag for this template.", since = "4.20.0") private String templateTag; + @Parameter(name = ApiConstants.FOR_CKS, type = CommandType.BOOLEAN, + description = "indicates that the template can be used for deployment of CKS clusters", + since = "4.21.0") + private Boolean forCks; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -63,6 +68,10 @@ public class UpdateTemplateCmd extends BaseUpdateTemplateOrIsoCmd implements Use return templateTag; } + public Boolean getForCks() { + return forCks; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java new file mode 100644 index 00000000000..c002bd226a0 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/BaseRegisterUserDataCmd.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.network.NetworkModel; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public abstract class BaseRegisterUserDataCmd extends BaseCmd { + + @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of the user data") + private String name; + + //Owner information + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the user data. Must be used with domainId.") + private String accountName; + + @Parameter(name = ApiConstants.DOMAIN_ID, + type = CommandType.UUID, + entityType = DomainResponse.class, + description = "an optional domainId for the user data. If the account parameter is used, domainId must also be used.") + private Long domainId; + + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the user data") + private Long projectId; + + @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in user data content") + private String params; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getName() { + return name; + } + + public String getAccountName() { + return accountName; + } + + public Long getDomainId() { + return domainId; + } + + public Long getProjectId() { + return projectId; + } + + public String getParams() { + checkForVRMetadataFileNames(params); + return params; + } + + public void checkForVRMetadataFileNames(String params) { + if (StringUtils.isNotEmpty(params)) { + List keyValuePairs = new ArrayList<>(Arrays.asList(params.split(","))); + keyValuePairs.retainAll(NetworkModel.metadataFileNames); + if (!keyValuePairs.isEmpty()) { + throw new InvalidParameterValueException(String.format("Params passed here have a few virtual router metadata file names %s", keyValuePairs)); + } + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/DeleteCniConfigurationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/DeleteCniConfigurationCmd.java new file mode 100644 index 00000000000..8faa612d3d9 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/DeleteCniConfigurationCmd.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.user.Account; +import com.cloud.user.UserData; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +@APICommand(name = "deleteCniConfiguration", description = "Deletes a CNI Configuration", responseObject = SuccessResponse.class, entityType = {UserData.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class DeleteCniConfigurationCmd extends DeleteUserDataCmd { + + public static final Logger logger = LogManager.getLogger(DeleteCniConfigurationCmd.class.getName()); + + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + boolean result = _mgr.deleteCniConfiguration(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to delete CNI configuration"); + } + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + Long domainId = this.getDomainId(); + String accountName = this.getAccountName(); + if ((account == null || _accountService.isAdmin(account.getId())) && (domainId != null && accountName != null)) { + Account userAccount = _responseGenerator.findAccountByNameDomain(accountName, domainId); + if (userAccount != null) { + return userAccount.getId(); + } + } + + if (account != null) { + return account.getId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java new file mode 100644 index 00000000000..3a0c3f1168b --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListCniConfigurationCmd.java @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import com.cloud.user.UserData; +import com.cloud.utils.Pair; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.UserDataResponse; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +@APICommand(name = "listCniConfiguration", description = "List user data for CNI plugins", responseObject = UserDataResponse.class, entityType = {UserData.class}, + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class ListCniConfigurationCmd extends ListUserDataCmd { + public static final Logger logger = LogManager.getLogger(ListCniConfigurationCmd.class.getName()); + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + Pair, Integer> resultList = _mgr.listUserDatas(this, true); + List responses = new ArrayList<>(); + for (UserData result : resultList.first()) { + UserDataResponse r = _responseGenerator.createUserDataResponse(result); + r.setObjectName(ApiConstants.CNI_CONFIG); + responses.add(r); + } + + ListResponse response = new ListResponse<>(); + response.setResponses(responses, resultList.second()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } + + +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java index 64ab3ec3d70..16bf1e5c1e4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmd.java @@ -61,7 +61,7 @@ public class ListUserDataCmd extends BaseListProjectAndAccountResourcesCmd { @Override public void execute() { - Pair, Integer> resultList = _mgr.listUserDatas(this); + Pair, Integer> resultList = _mgr.listUserDatas(this, false); List responses = new ArrayList<>(); for (UserData result : resultList.first()) { UserDataResponse r = _responseGenerator.createUserDataResponse(result); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java new file mode 100644 index 00000000000..eb80da3be05 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterCniConfigurationCmd.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.userdata; + +import com.cloud.user.UserData; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.UserDataResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +@APICommand(name = "registerCniConfiguration", + description = "Register a CNI Configuration to be used with CKS cluster", + since = "4.21.0", + responseObject = SuccessResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class RegisterCniConfigurationCmd extends BaseRegisterUserDataCmd { + public static final Logger logger = LogManager.getLogger(RegisterCniConfigurationCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.CNI_CONFIG, type = CommandType.STRING, description = "CNI Configuration content to be registered as User data", length = 1048576) + private String cniConfig; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public String getCniConfig() { + return cniConfig; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + UserData result = _mgr.registerCniConfiguration(this); + UserDataResponse response = _responseGenerator.createUserDataResponse(result); + response.setResponseName(getCommandName()); + response.setObjectName(ApiConstants.CNI_CONFIG); + setResponseObject(response); + } + + @Override + public long getEntityOwnerId() { + Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); + if (accountId == null) { + return CallContext.current().getCallingAccount().getId(); + } + + return accountId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java index 41d865d678c..fabf8827796 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java @@ -16,30 +16,20 @@ // under the License. package org.apache.cloudstack.api.command.user.userdata; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; -import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.context.CallContext; -import org.apache.commons.lang3.StringUtils; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.NetworkRuleConflictException; import com.cloud.exception.ResourceAllocationException; import com.cloud.exception.ResourceUnavailableException; -import com.cloud.network.NetworkModel; import com.cloud.user.UserData; @APICommand(name = "registerUserData", @@ -49,89 +39,28 @@ import com.cloud.user.UserData; requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) -public class RegisterUserDataCmd extends BaseCmd { +public class RegisterUserDataCmd extends BaseRegisterUserDataCmd { ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "Name of the userdata") - private String name; + @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, required = true, description = "User data content", length = 1048576) + protected String userData; - //Owner information - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the userdata. Must be used with domainId.") - private String accountName; - - @Parameter(name = ApiConstants.DOMAIN_ID, - type = CommandType.UUID, - entityType = DomainResponse.class, - description = "an optional domainId for the userdata. If the account parameter is used, domainId must also be used.") - private Long domainId; - - @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the userdata") - private Long projectId; - - @Parameter(name = ApiConstants.USER_DATA, - type = CommandType.STRING, - required = true, - description = "Base64 encoded userdata content. " + - "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " + - "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " + - "You also need to change vm.userdata.max.length value", - length = 1048576) - private String userData; - - @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in userdata content") - private String params; - - - ///////////////////////////////////////////////////// - /////////////////// Accessors /////////////////////// - ///////////////////////////////////////////////////// - - public String getName() { - return name; - } - - public String getAccountName() { - return accountName; - } - - public Long getDomainId() { - return domainId; - } - - public Long getProjectId() { - return projectId; - } public String getUserData() { return userData; } - public String getParams() { - checkForVRMetadataFileNames(params); - return params; - } - - public void checkForVRMetadataFileNames(String params) { - if (StringUtils.isNotEmpty(params)) { - List keyValuePairs = new ArrayList<>(Arrays.asList(params.split(","))); - keyValuePairs.retainAll(NetworkModel.metadataFileNames); - if (!keyValuePairs.isEmpty()) { - throw new InvalidParameterValueException(String.format("Params passed here have a few virtual router metadata file names %s", keyValuePairs)); - } - } - } - ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @Override public long getEntityOwnerId() { - Long accountId = _accountService.finalyzeAccountId(accountName, domainId, projectId, true); + Long accountId = _accountService.finalyzeAccountId(getAccountName(), getDomainId(), getProjectId(), true); if (accountId == null) { return CallContext.current().getCallingAccount().getId(); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index eee16287eea..9fc4258c6d9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -16,10 +16,15 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; +import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -54,6 +59,7 @@ import com.cloud.cpu.CPU; import com.cloud.exception.InvalidParameterValueException; import com.cloud.server.ResourceIcon; import com.cloud.server.ResourceTag; +import com.cloud.storage.GuestOS; import com.cloud.vm.VirtualMachine; @@ -331,22 +337,75 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements setResponseObject(response); } - protected void updateVMResponse(List response) { - for (UserVmResponse vmResponse : response) { - ResourceIcon resourceIcon = resourceIconManager.getByResourceTypeAndUuid(ResourceTag.ResourceObjectType.UserVm, vmResponse.getId()); - if (resourceIcon == null) { - ResourceTag.ResourceObjectType type = ResourceTag.ResourceObjectType.Template; - String uuid = vmResponse.getTemplateId(); - if (vmResponse.getIsoId() != null) { - uuid = vmResponse.getIsoId(); - type = ResourceTag.ResourceObjectType.ISO; - } - resourceIcon = resourceIconManager.getByResourceTypeAndUuid(type, uuid); - if (resourceIcon == null) { - continue; - } + protected Map getResourceIconsUsingOsCategory(List responses) { + Set guestOsIds = responses.stream().map(UserVmResponse::getGuestOsId).collect(Collectors.toSet()); + List guestOSList = _entityMgr.listByUuids(GuestOS.class, guestOsIds); + Map guestOSMap = guestOSList.stream() + .collect(Collectors.toMap(GuestOS::getUuid, Function.identity())); + Set guestOsCategoryIds = guestOSMap.values().stream() + .map(GuestOS::getCategoryId) + .collect(Collectors.toSet()); + Map guestOsCategoryIcons = + resourceIconManager.getByResourceTypeAndIds(ResourceTag.ResourceObjectType.GuestOsCategory, + guestOsCategoryIds); + Map vmIcons = new HashMap<>(); + for (UserVmResponse response : responses) { + GuestOS guestOS = guestOSMap.get(response.getGuestOsId()); + if (guestOS != null) { + vmIcons.put(response.getId(), guestOsCategoryIcons.get(guestOS.getCategoryId())); } - ResourceIconResponse iconResponse = _responseGenerator.createResourceIconResponse(resourceIcon); + } + return vmIcons; + } + + protected Map getResourceIconsForUsingTemplateIso(List responses) { + Map vmTemplateIsoIdMap = new HashMap<>(); + Set templateUuids = new HashSet<>(); + Set isoUuids = new HashSet<>(); + for (UserVmResponse vmResponse : responses) { + if (vmResponse.getTemplateId() != null) { + templateUuids.add(vmResponse.getTemplateId()); + vmTemplateIsoIdMap.put(vmResponse.getId(), vmResponse.getTemplateId()); + } + if (vmResponse.getIsoId() != null) { + isoUuids.add(vmResponse.getIsoId()); + vmTemplateIsoIdMap.put(vmResponse.getId(), vmResponse.getIsoId()); + } + } + Map templateOrIsoIcons = resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.Template, templateUuids); + templateOrIsoIcons.putAll(resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.ISO, isoUuids)); + Map vmIcons = new HashMap<>(); + List noTemplateIsoIconResponses = new ArrayList<>(); + for (UserVmResponse response : responses) { + String uuid = vmTemplateIsoIdMap.get(response.getId()); + if (StringUtils.isNotBlank(uuid) && templateOrIsoIcons.containsKey(uuid)) { + vmIcons.put(response.getId(), + templateOrIsoIcons.get(vmTemplateIsoIdMap.get(response.getId()))); + continue; + } + noTemplateIsoIconResponses.add(response); + } + vmIcons.putAll(getResourceIconsUsingOsCategory(noTemplateIsoIconResponses)); + return vmIcons; + } + + protected void updateVMResponse(List responses) { + if (CollectionUtils.isEmpty(responses)) { + return; + } + Set vmUuids = responses.stream().map(UserVmResponse::getId).collect(Collectors.toSet()); + Map vmIcons = resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.UserVm, vmUuids); + List noVmIconResponses = responses + .stream() + .filter(r -> !vmIcons.containsKey(r.getId())) + .collect(Collectors.toList()); + vmIcons.putAll(getResourceIconsForUsingTemplateIso(noVmIconResponses)); + for (UserVmResponse vmResponse : responses) { + ResourceIcon icon = vmIcons.get(vmResponse.getId()); + if (icon == null) { + continue; + } + ResourceIconResponse iconResponse = _responseGenerator.createResourceIconResponse(icon); vmResponse.setResourceIconResponse(iconResponse); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index 2f62d0d7210..08b7b9911b1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.user.vpc; +import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.cloudstack.acl.RoleType; @@ -125,6 +126,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the VPC tiers") private Long asNumber; + @Parameter(name=ApiConstants.USE_VIRTUAL_ROUTER_IP_RESOLVER, type=CommandType.BOOLEAN, + description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2") + private Boolean useVrIpResolver; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -205,6 +210,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { return asNumber; } + public boolean getUseVrIpResolver() { + return BooleanUtils.toBoolean(useVrIpResolver); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/GuestOSCategoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/GuestOSCategoryResponse.java index 7872bf22085..495e349856a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/GuestOSCategoryResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/GuestOSCategoryResponse.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.api.response; +import java.util.Date; + import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.ApiConstants; @@ -26,7 +28,7 @@ import com.cloud.serializer.Param; import com.cloud.storage.GuestOsCategory; @EntityReference(value = GuestOsCategory.class) -public class GuestOSCategoryResponse extends BaseResponse { +public class GuestOSCategoryResponse extends BaseResponse implements SetResourceIconResponse { @SerializedName(ApiConstants.ID) @Param(description = "the ID of the OS category") private String id; @@ -35,6 +37,18 @@ public class GuestOSCategoryResponse extends BaseResponse { @Param(description = "the name of the OS category") private String name; + @SerializedName(ApiConstants.IS_FEATURED) + @Param(description = "Whether the OS category is featured", since = "4.21.0") + private Boolean featured; + + @SerializedName(ApiConstants.RESOURCE_ICON) + @Param(description = "Base64 string representation of the resource icon", since = "4.21.0") + private ResourceIconResponse resourceIconResponse; + + @SerializedName(ApiConstants.CREATED) + @Param(description = "Date when the OS category was created." ) + private Date created; + public String getId() { return id; } @@ -50,4 +64,17 @@ public class GuestOSCategoryResponse extends BaseResponse { public void setName(String name) { this.name = name; } + + public void setFeatured(Boolean featured) { + this.featured = featured; + } + + @Override + public void setResourceIconResponse(ResourceIconResponse resourceIconResponse) { + this.resourceIconResponse = resourceIconResponse; + } + + public void setCreated(Date created) { + this.created = created; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java new file mode 100644 index 00000000000..cef5cdae2f4 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/KubernetesUserVmResponse.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.cloud.network.router.VirtualRouter; +import com.cloud.serializer.Param; +import com.cloud.uservm.UserVm; +import com.cloud.vm.VirtualMachine; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.EntityReference; + +@EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class}) +public class KubernetesUserVmResponse extends UserVmResponse { + @SerializedName(ApiConstants.IS_EXTERNAL_NODE) + @Param(description = "If the VM is an externally added node") + private boolean isExternalNode; + + @SerializedName(ApiConstants.IS_ETCD_NODE) + @Param(description = "If the VM is an etcd node") + private boolean isEtcdNode; + + @SerializedName(ApiConstants.KUBERNETES_NODE_VERSION) + @Param(description = "Kubernetes version of the node") + private String nodeVersion; + + + public void setExternalNode(boolean externalNode) { + isExternalNode = externalNode; + } + + public void setEtcdNode(boolean etcdNode) { + isEtcdNode = etcdNode; + } + + public void setNodeVersion(String nodeVersion) { this.nodeVersion = nodeVersion;} +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java index 98e96091d8c..57970368d7e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/TemplateResponse.java @@ -93,6 +93,8 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements @Param(description = "the name of the OS type for this template.") private String osTypeName; + private transient Long osTypeCategoryId; + @SerializedName(ApiConstants.ACCOUNT_ID) @Param(description = "the account id to which the template belongs") private String accountId; @@ -208,6 +210,11 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements since = "4.15") private Boolean deployAsIs; + @SerializedName(ApiConstants.FOR_CKS) + @Param(description = "If true it indicates that the template can be used for CKS cluster deployments", + since = "4.21.0") + private Boolean forCks; + @SerializedName(ApiConstants.DEPLOY_AS_IS_DETAILS) @Param(description = "VMware only: additional key/value details tied with deploy-as-is template", since = "4.15") @@ -285,6 +292,14 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements this.osTypeName = osTypeName; } + public Long getOsTypeCategoryId() { + return osTypeCategoryId; + } + + public void setOsTypeCategoryId(Long osTypeCategoryId) { + this.osTypeCategoryId = osTypeCategoryId; + } + public void setId(String id) { this.id = id; } @@ -453,6 +468,10 @@ public class TemplateResponse extends BaseResponseWithTagInformation implements this.deployAsIs = deployAsIs; } + public void setForCks(Boolean forCks) { + this.forCks = forCks; + } + public void setParentTemplateId(String parentTemplateId) { this.parentTemplateId = parentTemplateId; } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCategoryCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCategoryCmdTest.java new file mode 100644 index 00000000000..32230056b6a --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/AddGuestOsCategoryCmdTest.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.guest; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class AddGuestOsCategoryCmdTest { + + @Test + public void testGetArch() { + AddGuestOsCategoryCmd cmd = new AddGuestOsCategoryCmd(); + Assert.assertNull(cmd.getName()); + String name = "Name"; + ReflectionTestUtils.setField(cmd, "name", name); + Assert.assertEquals(name, cmd.getName()); + } + + @Test + public void testIsFeatured() { + AddGuestOsCategoryCmd cmd = new AddGuestOsCategoryCmd(); + Assert.assertFalse(cmd.isFeatured()); + ReflectionTestUtils.setField(cmd, "featured", false); + Assert.assertFalse(cmd.isFeatured()); + ReflectionTestUtils.setField(cmd, "featured", true); + Assert.assertTrue(cmd.isFeatured()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCategoryCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCategoryCmdTest.java new file mode 100644 index 00000000000..3f6a943b3fe --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCategoryCmdTest.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.guest; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class UpdateGuestOsCategoryCmdTest { + + @Test + public void testGetArch() { + UpdateGuestOsCategoryCmd cmd = new UpdateGuestOsCategoryCmd(); + Assert.assertNull(cmd.getName()); + String name = "Name"; + ReflectionTestUtils.setField(cmd, "name", name); + Assert.assertEquals(name, cmd.getName()); + } + + @Test + public void testIsFeatured() { + UpdateGuestOsCategoryCmd cmd = new UpdateGuestOsCategoryCmd(); + Assert.assertNull(cmd.isFeatured()); + ReflectionTestUtils.setField(cmd, "featured", false); + Assert.assertFalse(cmd.isFeatured()); + ReflectionTestUtils.setField(cmd, "featured", true); + Assert.assertTrue(cmd.isFeatured()); + } + + @Test + public void testGetSortKey() { + UpdateGuestOsCategoryCmd cmd = new UpdateGuestOsCategoryCmd(); + Assert.assertNull(cmd.getSortKey()); + Integer sortKey = 100; + ReflectionTestUtils.setField(cmd, "sortKey", sortKey); + Assert.assertEquals(sortKey, cmd.getSortKey()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmdTest.java new file mode 100644 index 00000000000..f03a791776e --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/guest/UpdateGuestOsCmdTest.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.guest; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +@RunWith(MockitoJUnitRunner.class) +public class UpdateGuestOsCmdTest { + + + @Test + public void testGetZoneId() { + UpdateGuestOsCmd cmd = new UpdateGuestOsCmd(); + Assert.assertNull(cmd.getOsCategoryId()); + Long osCategoryId = 100L; + ReflectionTestUtils.setField(cmd, "osCategoryId", osCategoryId); + Assert.assertEquals(osCategoryId, cmd.getOsCategoryId()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmdTest.java new file mode 100644 index 00000000000..f417dc5f876 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCategoriesCmdTest.java @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.guest; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.cpu.CPU; + +@RunWith(MockitoJUnitRunner.class) +public class ListGuestOsCategoriesCmdTest { + + @Test + public void testIsFeatured() { + ListGuestOsCategoriesCmd cmd = new ListGuestOsCategoriesCmd(); + Assert.assertNull(cmd.isFeatured()); + ReflectionTestUtils.setField(cmd, "featured", false); + Assert.assertFalse(cmd.isFeatured()); + ReflectionTestUtils.setField(cmd, "featured", true); + Assert.assertTrue(cmd.isFeatured()); + } + + @Test + public void testIsIso() { + ListGuestOsCategoriesCmd cmd = new ListGuestOsCategoriesCmd(); + Assert.assertNull(cmd.isIso()); + ReflectionTestUtils.setField(cmd, "iso", false); + Assert.assertFalse(cmd.isIso()); + ReflectionTestUtils.setField(cmd, "iso", true); + Assert.assertTrue(cmd.isIso()); + } + + @Test + public void testIsVnf() { + ListGuestOsCategoriesCmd cmd = new ListGuestOsCategoriesCmd(); + Assert.assertNull(cmd.isVnf()); + ReflectionTestUtils.setField(cmd, "vnf", false); + Assert.assertFalse(cmd.isVnf()); + ReflectionTestUtils.setField(cmd, "vnf", true); + Assert.assertTrue(cmd.isVnf()); + } + + @Test + public void testGetZoneId() { + ListGuestOsCategoriesCmd cmd = new ListGuestOsCategoriesCmd(); + Assert.assertNull(cmd.getZoneId()); + Long zoneId = 100L; + ReflectionTestUtils.setField(cmd, "zoneId", zoneId); + Assert.assertEquals(zoneId, cmd.getZoneId()); + } + + @Test + public void testGetArch() { + ListGuestOsCategoriesCmd cmd = new ListGuestOsCategoriesCmd(); + Assert.assertNull(cmd.getArch()); + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + ReflectionTestUtils.setField(cmd, "arch", arch.getType()); + Assert.assertEquals(arch, cmd.getArch()); + } + + @Test + public void testIsShowIcon() { + ListGuestOsCategoriesCmd cmd = new ListGuestOsCategoriesCmd(); + Assert.assertFalse(cmd.isShowIcon()); + ReflectionTestUtils.setField(cmd, "showIcon", false); + Assert.assertFalse(cmd.isShowIcon()); + ReflectionTestUtils.setField(cmd, "showIcon", true); + Assert.assertTrue(cmd.isShowIcon()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java index 3f47a078445..8b7db292462 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/userdata/ListUserDataCmdTest.java @@ -68,7 +68,7 @@ public class ListUserDataCmdTest { Pair, Integer> result = new Pair, Integer>(userDataList, 1); UserDataResponse userDataResponse = Mockito.mock(UserDataResponse.class); - Mockito.when(_mgr.listUserDatas(cmd)).thenReturn(result); + Mockito.when(_mgr.listUserDatas(cmd, false)).thenReturn(result); Mockito.when(_responseGenerator.createUserDataResponse(userData)).thenReturn(userDataResponse); cmd.execute(); @@ -82,7 +82,7 @@ public class ListUserDataCmdTest { List userDataList = new ArrayList(); Pair, Integer> result = new Pair, Integer>(userDataList, 0); - Mockito.when(_mgr.listUserDatas(cmd)).thenReturn(result); + Mockito.when(_mgr.listUserDatas(cmd, false)).thenReturn(result); cmd.execute(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmdTest.java new file mode 100644 index 00000000000..48b41a47fff --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmdTest.java @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.anySet; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import org.apache.cloudstack.api.ResponseGenerator; +import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import com.cloud.server.ResourceIcon; +import com.cloud.server.ResourceIconManager; +import com.cloud.server.ResourceTag; +import com.cloud.storage.GuestOS; +import com.cloud.utils.db.EntityManager; + +public class ListVMsCmdTest { + + EntityManager _entityMgr; + ResourceIconManager resourceIconManager; + ResponseGenerator _responseGenerator; + + ListVMsCmd cmd; + + @Before + public void setup() { + _entityMgr = mock(EntityManager.class); + resourceIconManager = mock(ResourceIconManager.class); + _responseGenerator = mock(ResponseGenerator.class); + cmd = spy(ListVMsCmd.class); + cmd._entityMgr = _entityMgr; + cmd.resourceIconManager = resourceIconManager; + cmd._responseGenerator = _responseGenerator; + } + + @Test + public void testUpdateVMResponse_withMixedIcons() { + String vm1Uuid = UUID.randomUUID().toString(); + UserVmResponse vm1 = mock(UserVmResponse.class); + when(vm1.getId()).thenReturn(vm1Uuid); + String vm2Uuid = UUID.randomUUID().toString(); + UserVmResponse vm2 = mock(UserVmResponse.class); + when(vm2.getId()).thenReturn(vm2Uuid); + List responses = Arrays.asList(vm1, vm2); + ResourceIcon icon1 = mock(ResourceIcon.class); + ResourceIcon icon2 = mock(ResourceIcon.class); + Map initialIcons = new HashMap<>(); + initialIcons.put(vm1Uuid, icon1); + when(resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.UserVm, Set.of(vm1Uuid, vm2Uuid))) + .thenReturn(initialIcons); + Map fallbackIcons = Map.of(vm2Uuid, icon2); + doReturn(fallbackIcons).when(cmd).getResourceIconsForUsingTemplateIso(anyList()); + ResourceIconResponse iconResponse1 = new ResourceIconResponse(); + ResourceIconResponse iconResponse2 = new ResourceIconResponse(); + when(_responseGenerator.createResourceIconResponse(icon1)).thenReturn(iconResponse1); + when(_responseGenerator.createResourceIconResponse(icon2)).thenReturn(iconResponse2); + cmd.updateVMResponse(responses); + verify(vm1).setResourceIconResponse(iconResponse1); + verify(vm2).setResourceIconResponse(iconResponse2); + } + + @Test + public void testUpdateVMResponse_withEmptyList() { + cmd.updateVMResponse(Collections.emptyList()); + verify(resourceIconManager, never()).getByResourceTypeAndIds(Mockito.any(), Mockito.anyCollection()); + } + + @Test + public void testGetResourceIconsForUsingTemplateIso_withValidData() { + String vm1Uuid = UUID.randomUUID().toString(); + String template1Uuid = UUID.randomUUID().toString(); + UserVmResponse vm1 = mock(UserVmResponse.class); + when(vm1.getId()).thenReturn(vm1Uuid); + when(vm1.getTemplateId()).thenReturn(template1Uuid); + when(vm1.getIsoId()).thenReturn(null); + String vm2Uuid = UUID.randomUUID().toString(); + String iso2Uuid = UUID.randomUUID().toString(); + UserVmResponse vm2 = mock(UserVmResponse.class); + when(vm2.getId()).thenReturn(vm2Uuid); + when(vm2.getTemplateId()).thenReturn(null); + when(vm2.getIsoId()).thenReturn(iso2Uuid); + List responses = Arrays.asList(vm1, vm2); + Map templateIcons = new HashMap<>(); + templateIcons.put(template1Uuid, mock(ResourceIcon.class)); + Map isoIcons = new HashMap<>(); + isoIcons.put(iso2Uuid, mock(ResourceIcon.class)); + when(resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.Template, Set.of(template1Uuid))) + .thenReturn(templateIcons); + when(resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.ISO, Set.of(iso2Uuid))) + .thenReturn(isoIcons); + doReturn(Collections.emptyMap()).when(cmd).getResourceIconsUsingOsCategory(anyList()); + Map result = cmd.getResourceIconsForUsingTemplateIso(responses); + assertEquals(2, result.size()); + assertTrue(result.containsKey(vm1Uuid)); + assertTrue(result.containsKey(vm2Uuid)); + assertEquals(templateIcons.get(template1Uuid), result.get(vm1Uuid)); + assertEquals(isoIcons.get(iso2Uuid), result.get(vm2Uuid)); + } + + @Test + public void testGetResourceIconsForUsingTemplateIso_withMissingIcons() { + String vm1Uuid = UUID.randomUUID().toString(); + String template1Uuid = UUID.randomUUID().toString(); + UserVmResponse vm1 = mock(UserVmResponse.class); + when(vm1.getId()).thenReturn(vm1Uuid); + when(vm1.getTemplateId()).thenReturn(template1Uuid); + when(vm1.getIsoId()).thenReturn(null); + List responses = List.of(vm1); + when(resourceIconManager.getByResourceTypeAndUuids(eq(ResourceTag.ResourceObjectType.Template), anySet())) + .thenReturn(Collections.emptyMap()); + when(resourceIconManager.getByResourceTypeAndUuids(eq(ResourceTag.ResourceObjectType.ISO), anySet())) + .thenReturn(Collections.emptyMap()); + Map fallbackIcons = Map.of(vm1Uuid, mock(ResourceIcon.class)); + doReturn(fallbackIcons).when(cmd).getResourceIconsUsingOsCategory(anyList()); + Map result = cmd.getResourceIconsForUsingTemplateIso(responses); + assertEquals(1, result.size()); + assertEquals(fallbackIcons.get("vm1"), result.get("vm1")); + } + + @Test + public void testGetResourceIconsUsingOsCategory_withValidData() { + String vm1Uuid = UUID.randomUUID().toString(); + String os1Uuid = UUID.randomUUID().toString(); + UserVmResponse vm1 = mock(UserVmResponse.class); + when(vm1.getGuestOsId()).thenReturn(os1Uuid); + when(vm1.getId()).thenReturn(vm1Uuid); + String vm2Uuid = UUID.randomUUID().toString(); + String os2Uuid = UUID.randomUUID().toString(); + UserVmResponse vm2 = mock(UserVmResponse.class); + when(vm2.getGuestOsId()).thenReturn(os2Uuid); + when(vm2.getId()).thenReturn(vm2Uuid); + List responses = Arrays.asList(vm1, vm2); + GuestOS guestOS1 = mock(GuestOS.class); + when(guestOS1.getUuid()).thenReturn(os1Uuid); + when(guestOS1.getCategoryId()).thenReturn(10L); + GuestOS guestOS2 = mock(GuestOS.class); + when(guestOS2.getUuid()).thenReturn(os2Uuid); + when(guestOS2.getCategoryId()).thenReturn(20L); + when(_entityMgr.listByUuids(eq(GuestOS.class), anySet())) + .thenReturn(Arrays.asList(guestOS1, guestOS2)); + ResourceIcon icon1 = mock(ResourceIcon.class); + ResourceIcon icon2 = mock(ResourceIcon.class); + Map categoryIcons = new HashMap<>(); + categoryIcons.put(10L, icon1); + categoryIcons.put(20L, icon2); + when(resourceIconManager.getByResourceTypeAndIds(eq(ResourceTag.ResourceObjectType.GuestOsCategory), anySet())) + .thenReturn(categoryIcons); + Map result = cmd.getResourceIconsUsingOsCategory(responses); + assertEquals(2, result.size()); + assertEquals(icon1, result.get(vm1Uuid)); + assertEquals(icon2, result.get(vm2Uuid)); + } + + @Test + public void testGetResourceIconsUsingOsCategory_missingGuestOS() { + String vm1Uuid = UUID.randomUUID().toString(); + String os1Uuid = UUID.randomUUID().toString(); + UserVmResponse vm1 = mock(UserVmResponse.class); + when(vm1.getGuestOsId()).thenReturn(vm1Uuid); + when(vm1.getId()).thenReturn(os1Uuid); + List responses = Collections.singletonList(vm1); + when(_entityMgr.listByUuids(eq(GuestOS.class), anySet())) + .thenReturn(Collections.emptyList()); + Map result = cmd.getResourceIconsUsingOsCategory(responses); + assertTrue(result.isEmpty()); + } + + @Test + public void testGetResourceIconsUsingOsCategory_missingIcon() { + UserVmResponse vm1 = mock(UserVmResponse.class); + String vmUuid = UUID.randomUUID().toString(); + String osUuid = UUID.randomUUID().toString(); + when(vm1.getGuestOsId()).thenReturn(osUuid); + when(vm1.getId()).thenReturn(vmUuid); + List responses = Collections.singletonList(vm1); + GuestOS guestOS1 = mock(GuestOS.class); + when(guestOS1.getCategoryId()).thenReturn(10L); + when(guestOS1.getUuid()).thenReturn(osUuid); + when(_entityMgr.listByUuids(eq(GuestOS.class), anySet())) + .thenReturn(Collections.singletonList(guestOS1)); + when(resourceIconManager.getByResourceTypeAndIds(eq(ResourceTag.ResourceObjectType.GuestOsCategory), anySet())) + .thenReturn(Collections.emptyMap()); + Map result = cmd.getResourceIconsUsingOsCategory(responses); + assertTrue(result.containsKey(vmUuid)); + assertNull(result.get(vmUuid)); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java b/core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java new file mode 100644 index 00000000000..16942bb05d4 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/HandleCksIsoCommand.java @@ -0,0 +1,34 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +import com.cloud.agent.api.routing.NetworkElementCommand; + +public class HandleCksIsoCommand extends NetworkElementCommand { + + private boolean mountCksIso; + + public HandleCksIsoCommand(boolean mountCksIso) { + this.mountCksIso = mountCksIso; + } + + public boolean isMountCksIso() { + return mountCksIso; + } +} diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java index f9ea3e05e97..7bfbf786e9b 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VRScripts.java @@ -82,5 +82,8 @@ public class VRScripts { public static final String VR_UPDATE_INTERFACE_CONFIG = "update_interface_config.sh"; public static final String ROUTER_FILESYSTEM_WRITABLE_CHECK = "filesystem_writable_check.py"; + + // CKS ISO mount + public static final String CKS_ISO_MOUNT_SERVE = "cks_iso.sh"; public static final String MANAGE_SERVICE = "manage_service.sh"; } diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java index 4afac9b43cb..bd632632ae8 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResource.java @@ -34,6 +34,7 @@ import java.util.concurrent.locks.ReentrantLock; import javax.naming.ConfigurationException; +import com.cloud.agent.api.HandleCksIsoCommand; import org.apache.cloudstack.agent.routing.ManageServiceCommand; import com.cloud.agent.api.routing.UpdateNetworkCommand; import com.cloud.agent.api.to.IpAddressTO; @@ -145,6 +146,10 @@ public class VirtualRoutingResource { return execute((UpdateNetworkCommand) cmd); } + if (cmd instanceof HandleCksIsoCommand) { + return execute((HandleCksIsoCommand) cmd); + } + if (cmd instanceof ManageServiceCommand) { return execute((ManageServiceCommand) cmd); } @@ -176,6 +181,13 @@ public class VirtualRoutingResource { } } + protected Answer execute(final HandleCksIsoCommand cmd) { + String routerIp = getRouterSshControlIp(cmd); + logger.info("Attempting to mount CKS ISO on Virtual Router"); + ExecutionResult result = _vrDeployer.executeInVR(routerIp, VRScripts.CKS_ISO_MOUNT_SERVE, String.valueOf(cmd.isMountCksIso())); + return new Answer(cmd, result.isSuccess(), result.getDetails()); + } + private Answer execute(final SetupKeyStoreCommand cmd) { final String args = String.format("/usr/local/cloud/systemvm/conf/agent.properties " + "/usr/local/cloud/systemvm/conf/%s " + diff --git a/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java b/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java index 32f436434c1..864a3e22eb3 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java @@ -29,13 +29,15 @@ public class SetupMSListCommand extends Command { private List avoidMsList; private String lbAlgorithm; private Long lbCheckInterval; + private Boolean triggerHostLb; - public SetupMSListCommand(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) { + public SetupMSListCommand(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final Boolean triggerHostLb) { super(); this.msList = msList; this.avoidMsList = avoidMsList; this.lbAlgorithm = lbAlgorithm; this.lbCheckInterval = lbCheckInterval; + this.triggerHostLb = triggerHostLb; } public List getMsList() { @@ -54,9 +56,12 @@ public class SetupMSListCommand extends Command { return lbCheckInterval; } + public boolean getTriggerHostLb() { + return triggerHostLb; + } + @Override public boolean executeInSequence() { return false; } - } diff --git a/debian/rules b/debian/rules index e7ff6759d44..d178afa6730 100755 --- a/debian/rules +++ b/debian/rules @@ -70,6 +70,7 @@ override_dh_auto_install: mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/lib mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/setup mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm + mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf mkdir $(DESTDIR)/var/log/$(PACKAGE)/management mkdir $(DESTDIR)/var/cache/$(PACKAGE)/management mkdir $(DESTDIR)/var/log/$(PACKAGE)/ipallocator @@ -83,6 +84,7 @@ override_dh_auto_install: cp client/target/cloud-client-ui-$(VERSION).jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/cloudstack-$(VERSION).jar cp client/target/lib/*jar $(DESTDIR)/usr/share/$(PACKAGE)-management/lib/ cp -r engine/schema/dist/systemvm-templates/* $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/ + cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/ rm -rf $(DESTDIR)/usr/share/$(PACKAGE)-management/templates/systemvm/md5sum.txt # Bundle cmk in cloudstack-management @@ -95,6 +97,12 @@ override_dh_auto_install: chmod 0440 $(DESTDIR)/$(SYSCONFDIR)/sudoers.d/$(PACKAGE) install -D client/target/utilities/bin/cloud-update-xenserver-licenses $(DESTDIR)/usr/bin/cloudstack-update-xenserver-licenses + + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/etcd-node.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-control-node-add.yml + install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml $(DESTDIR)/usr/share/$(PACKAGE)-management/cks/conf/k8s-node.yml + # Remove configuration in /ur/share/cloudstack-management/webapps/client/WEB-INF # This should all be in /etc/cloudstack/management ln -s ../../..$(SYSCONFDIR)/$(PACKAGE)/management $(DESTDIR)/usr/share/$(PACKAGE)-management/conf diff --git a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java index dd388d2a2d8..0aa5805b160 100644 --- a/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java +++ b/engine/components-api/src/main/java/com/cloud/agent/AgentManager.java @@ -171,5 +171,5 @@ public interface AgentManager { void propagateChangeToAgents(Map params); - boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs); + boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 585c479f65f..dc7852ed82b 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -62,6 +62,7 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.ThreadContext; @@ -273,8 +274,6 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl _executor = new ThreadPoolExecutor(agentTaskThreads, agentTaskThreads, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), new NamedThreadFactory("AgentTaskPool")); - initConnectExecutor(); - maxConcurrentNewAgentConnections = RemoteAgentMaxConcurrentNewConnections.value(); _connection = new NioServer("AgentManager", Port.value(), Workers.value() + 10, @@ -803,11 +802,25 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl Map detailsMap = readyAnswer.getDetailsMap(); if (detailsMap != null) { String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); + String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); + String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (uefiEnabled != null) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { _hostDao.loadDetails(host); + boolean updateNeeded = false; if (!uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { host.getDetails().put(Host.HOST_UEFI_ENABLE, uefiEnabled); + updateNeeded = true; + } + if (StringUtils.isNotBlank(virtv2vVersion) && !virtv2vVersion.equals(host.getDetails().get(Host.HOST_VIRTV2V_VERSION))) { + host.getDetails().put(Host.HOST_VIRTV2V_VERSION, virtv2vVersion); + updateNeeded = true; + } + if (StringUtils.isNotBlank(ovftoolVersion) && !ovftoolVersion.equals(host.getDetails().get(Host.HOST_OVFTOOL_VERSION))) { + host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); + updateNeeded = true; + } + if (updateNeeded) { _hostDao.saveDetails(host); } } @@ -828,6 +841,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl return true; } + initConnectExecutor(); startDirectlyConnectedHosts(false); if (_connection != null) { @@ -2193,7 +2207,7 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl } @Override - public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) { + public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) { return true; } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index 8795c8d428f..a7dca34f032 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -42,6 +42,7 @@ import javax.naming.ConfigurationException; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLEngine; +import com.cloud.resource.ResourceState; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.framework.config.ConfigDepot; import org.apache.cloudstack.framework.config.ConfigKey; @@ -431,10 +432,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust ch = connectToPeer(peer, ch); if (ch == null) { try { - logD(bytes, "Unable to route to peer: " + Request.parse(bytes)); + logD(bytes, "Unable to establish connection to route to peer: " + Request.parse(bytes)); } catch (ClassNotFoundException | UnsupportedVersionException e) { // Request.parse thrown exception when we try to log it, log as much as we can - logD(bytes, "Unable to route to peer, and Request.parse further caught exception" + e.getMessage()); + logD(bytes, "Unable to establish connection to route to peer, and Request.parse further caught exception" + e.getMessage()); } return false; } @@ -643,7 +644,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust final Link link = task.getLink(); if (Request.fromServer(data)) { - final AgentAttache agent = findAttache(hostId); if (Request.isControl(data)) { @@ -691,7 +691,6 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust cancel(Long.toString(Request.getManagementServerId(data)), hostId, Request.getSequence(data), e.getMessage()); } } else { - final long mgmtId = Request.getManagementServerId(data); if (mgmtId != -1 && mgmtId != _nodeId) { routeToPeer(Long.toString(mgmtId), data); @@ -1352,7 +1351,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust if (cmd instanceof PrepareForMaintenanceManagementServerHostCommand) { logger.debug("Received PrepareForMaintenanceManagementServerHostCommand - preparing for maintenance"); try { - managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm()); + managementServerMaintenanceManager.prepareForMaintenance(((PrepareForMaintenanceManagementServerHostCommand) cmd).getLbAlgorithm(), ((PrepareForMaintenanceManagementServerHostCommand) cmd).isForced()); return "Successfully prepared for maintenance"; } catch(CloudRuntimeException e) { return e.getMessage(); @@ -1399,14 +1398,14 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust } @Override - public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs) { + public boolean transferDirectAgentsFromMS(String fromMsUuid, long fromMsId, long timeoutDurationInMs, boolean excludeHostsInMaintenance) { if (timeoutDurationInMs <= 0) { logger.debug("Not transferring direct agents from management server node {} (id: {}) to other nodes, invalid timeout duration", fromMsId, fromMsUuid); return false; } long transferStartTimeInMs = System.currentTimeMillis(); - if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId))) { + if (CollectionUtils.isEmpty(getDirectAgentHosts(fromMsId, excludeHostsInMaintenance))) { logger.info("No direct agent hosts available on management server node {} (id: {}), to transfer", fromMsId, fromMsUuid); return true; } @@ -1421,7 +1420,7 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust int agentTransferFailedCount = 0; List dataCenterList = dcDao.listAll(); for (DataCenterVO dc : dataCenterList) { - List directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId()); + List directAgentHostsInDc = getDirectAgentHostsInDc(fromMsId, dc.getId(), excludeHostsInMaintenance); if (CollectionUtils.isEmpty(directAgentHostsInDc)) { continue; } @@ -1455,9 +1454,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust return (agentTransferFailedCount == 0); } - private List getDirectAgentHosts(long msId) { + private List getDirectAgentHosts(long msId, boolean excludeHostsInMaintenance) { List directAgentHosts = new ArrayList<>(); - List hosts = _hostDao.listHostsByMs(msId); + List statesToExclude = excludeHostsInMaintenance ? ResourceState.s_maintenanceStates : List.of(); + List hosts = _hostDao.listHostsByMsResourceState(msId, statesToExclude); for (HostVO host : hosts) { AgentAttache agent = findAttache(host.getId()); if (agent instanceof DirectAgentAttache) { @@ -1468,9 +1468,11 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust return directAgentHosts; } - private List getDirectAgentHostsInDc(long msId, long dcId) { + private List getDirectAgentHostsInDc(long msId, long dcId, boolean excludeHostsInMaintenance) { List directAgentHosts = new ArrayList<>(); - List hosts = _hostDao.listHostsByMsAndDc(msId, dcId); + // To exclude maintenance states use values from ResourceState as source of truth + List statesToExclude = excludeHostsInMaintenance ? ResourceState.s_maintenanceStates : List.of(); + List hosts = _hostDao.listHostsByMsDcResourceState(msId, dcId, statesToExclude); for (HostVO host : hosts) { AgentAttache agent = findAttache(host.getId()); if (agent instanceof DirectAgentAttache) { @@ -1506,6 +1508,10 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust public void onManagementServerCancelPreparingForMaintenance() { logger.debug("Management server cancel preparing for maintenance"); super.onManagementServerPreparingForMaintenance(); + + // needed for the case when Management Server in Preparing For Maintenance but didn't go to Maintenance state + // (where this variable will be reset) + _agentLbHappened = false; } @Override diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 2b8a23a1b51..090b019334f 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -177,14 +177,24 @@ public interface HostDao extends GenericDao, StateDao listHostsByMsAndDc(long msId, long dcId); + List listHostsByMsDcResourceState(long msId, long dcId, List excludedResourceStates); + List listHostsByMs(long msId); + List listHostsByMsResourceState(long msId, List excludedResourceStates); + /** - * Retrieves the number of hosts/agents this {@see ManagementServer} has responsibility over. - * @param msId the id of the {@see ManagementServer} - * @return the number of hosts/agents this {@see ManagementServer} has responsibility over + * Count Hosts by given Management Server, Host and Hypervisor Types, + * and exclude Hosts with given Resource States. + * + * @param msId Management Server Id + * @param excludedResourceStates Resource States to be excluded + * @param hostTypes Host Types + * @param hypervisorTypes Hypervisor Types + * @return Hosts count */ - int countByMs(long msId); + int countHostsByMsResourceStateTypeAndHypervisorType(long msId, List excludedResourceStates, + List hostTypes, List hypervisorTypes); /** * Retrieves the host ids/agents this {@see ManagementServer} has responsibility over. diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 61fa3edcf22..8f218841b07 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -72,6 +72,7 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.JoinBuilder.JoinType; +import com.cloud.utils.db.QueryBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Func; @@ -1600,6 +1601,17 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao return listBy(sc); } + @Override + public List listHostsByMsDcResourceState(long msId, long dcId, List excludedResourceStates) { + QueryBuilder sc = QueryBuilder.create(HostVO.class); + sc.and(sc.entity().getManagementServerId(), Op.EQ, msId); + sc.and(sc.entity().getDataCenterId(), Op.EQ, dcId); + if (CollectionUtils.isNotEmpty(excludedResourceStates)) { + sc.and(sc.entity().getResourceState(), Op.NIN, excludedResourceStates.toArray()); + } + return listBy(sc.create()); + } + @Override public List listHostsByMs(long msId) { SearchCriteria sc = ResponsibleMsSearch.create(); @@ -1608,10 +1620,32 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } @Override - public int countByMs(long msId) { - SearchCriteria sc = ResponsibleMsSearch.create(); - sc.setParameters("managementServerId", msId); - return getCount(sc); + public List listHostsByMsResourceState(long msId, List excludedResourceStates) { + QueryBuilder sc = QueryBuilder.create(HostVO.class); + sc.and(sc.entity().getManagementServerId(), Op.EQ, msId); + if (CollectionUtils.isNotEmpty(excludedResourceStates)) { + sc.and(sc.entity().getResourceState(), Op.NIN, excludedResourceStates.toArray()); + } + return listBy(sc.create()); + } + + @Override + public int countHostsByMsResourceStateTypeAndHypervisorType(long msId, + List excludedResourceStates, + List hostTypes, + List hypervisorTypes) { + QueryBuilder sc = QueryBuilder.create(HostVO.class); + sc.and(sc.entity().getManagementServerId(), Op.EQ, msId); + if (CollectionUtils.isNotEmpty(excludedResourceStates)) { + sc.and(sc.entity().getResourceState(), Op.NIN, excludedResourceStates.toArray()); + } + if (CollectionUtils.isNotEmpty(hostTypes)) { + sc.and(sc.entity().getType(), Op.IN, hostTypes.toArray()); + } + if (CollectionUtils.isNotEmpty(hypervisorTypes)) { + sc.and(sc.entity().getHypervisorType(), Op.IN, hypervisorTypes.toArray()); + } + return getCount(sc.create()); } @Override diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java index 056445225d0..7f322ae6c03 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDao.java @@ -75,5 +75,7 @@ public interface FirewallRulesDao extends GenericDao { void loadDestinationCidrs(FirewallRuleVO rule); + FirewallRuleVO findByNetworkIdAndPorts(long networkId, int startPort, int endPort); + List listRoutingIngressFirewallRules(long networkId); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java index a793a9172d4..27bf7ba6aa8 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/FirewallRulesDaoImpl.java @@ -48,6 +48,7 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i protected final SearchBuilder NotRevokedSearch; protected final SearchBuilder ReleaseSearch; protected SearchBuilder VmSearch; + protected SearchBuilder FirewallByPortsAndNetwork; protected final SearchBuilder SystemRuleSearch; protected final GenericSearchBuilder RulesByIpCount; protected final SearchBuilder RoutingFirewallRulesSearch; @@ -106,6 +107,12 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i RulesByIpCount.and("state", RulesByIpCount.entity().getState(), Op.EQ); RulesByIpCount.done(); + FirewallByPortsAndNetwork = createSearchBuilder(); + FirewallByPortsAndNetwork.and("networkId", FirewallByPortsAndNetwork.entity().getNetworkId(), Op.EQ); + FirewallByPortsAndNetwork.and("sourcePortStart", FirewallByPortsAndNetwork.entity().getSourcePortStart(), Op.EQ); + FirewallByPortsAndNetwork.and("sourcePortEnd", FirewallByPortsAndNetwork.entity().getSourcePortEnd(), Op.EQ); + FirewallByPortsAndNetwork.done(); + RoutingFirewallRulesSearch = createSearchBuilder(); RoutingFirewallRulesSearch.and("networkId", RoutingFirewallRulesSearch.entity().getNetworkId(), Op.EQ); RoutingFirewallRulesSearch.and("purpose", RoutingFirewallRulesSearch.entity().getPurpose(), Op.EQ); @@ -408,6 +415,16 @@ public class FirewallRulesDaoImpl extends GenericDaoBase i rule.setDestinationCidrsList(destCidrs); } + @Override + public FirewallRuleVO findByNetworkIdAndPorts(long networkId, int startPort, int endPort) { + SearchCriteria sc = FirewallByPortsAndNetwork.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("sourcePortStart", startPort); + sc.setParameters("sourcePortEnd", endPort); + + return findOneBy(sc); + } + @Override public List listRoutingIngressFirewallRules(long networkId) { SearchCriteria sc = RoutingFirewallRulesSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java index 8cd114b7fc4..a737f1b9a20 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDao.java @@ -47,5 +47,7 @@ public interface PortForwardingRulesDao extends GenericDao listByNetworkAndDestIpAddr(String ip4Address, long networkId); + + PortForwardingRuleVO findByNetworkAndPorts(long networkId, int startPort, int endPort); int expungeByVmList(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java index 1b3df06e1a2..637f47731b4 100644 --- a/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/rules/dao/PortForwardingRulesDaoImpl.java @@ -58,6 +58,8 @@ public class PortForwardingRulesDaoImpl extends GenericDaoBase sc = AllFieldsSearch.create(); + sc.setParameters("networkId", networkId); + sc.setParameters("sourcePortStart", startPort); + sc.setParameters("sourcePortEnd", endPort); + return findOneBy(sc); + } + @Override public int expungeByVmList(List vmIds, Long batchSize) { if (CollectionUtils.isEmpty(vmIds)) { diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java index e8ccc2ebcf1..e942eadb8ff 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java @@ -105,6 +105,9 @@ public class VpcVO implements Vpc { @Column(name = "ip6Dns2") String ip6Dns2; + @Column(name = "use_router_ip_resolver") + boolean useRouterIpResolver = false; + @Transient boolean rollingRestart = false; @@ -309,4 +312,13 @@ public class VpcVO implements Vpc { public String getIp6Dns2() { return ip6Dns2; } + + @Override + public boolean useRouterIpAsResolver() { + return useRouterIpResolver; + } + + public void setUseRouterIpResolver(boolean useRouterIpResolver) { + this.useRouterIpResolver = useRouterIpResolver; + } } diff --git a/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDao.java b/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDao.java index 3724e03d9d0..cee5cc15e11 100644 --- a/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDao.java +++ b/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDao.java @@ -22,10 +22,13 @@ import com.cloud.server.ResourceTag; import com.cloud.utils.db.GenericDao; import org.apache.cloudstack.api.response.ResourceIconResponse; +import java.util.Collection; import java.util.List; public interface ResourceIconDao extends GenericDao { ResourceIconResponse newResourceIconResponse(ResourceIcon resourceIconVO); ResourceIconVO findByResourceUuid(String resourceUuid, ResourceTag.ResourceObjectType resourceType); + List listByResourceTypeAndIds(ResourceTag.ResourceObjectType resourceType, Collection resourceIds); + List listByResourceTypeAndUuids(ResourceTag.ResourceObjectType resourceType, Collection resourceUuids); List listResourceIcons(List resourceUuids, ResourceTag.ResourceObjectType resourceType); } diff --git a/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDaoImpl.java b/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDaoImpl.java index 1ae01bfc1ec..49a1fe7a560 100644 --- a/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/resource/icon/dao/ResourceIconDaoImpl.java @@ -24,8 +24,10 @@ import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.commons.collections.CollectionUtils; import java.util.ArrayList; +import java.util.Collection; import java.util.List; public class ResourceIconDaoImpl extends GenericDaoBase implements ResourceIconDao { @@ -58,11 +60,36 @@ public class ResourceIconDaoImpl extends GenericDaoBase im } @Override - public List listResourceIcons(List resourceUuids, ResourceTag.ResourceObjectType resourceType) { + public List listByResourceTypeAndIds(ResourceTag.ResourceObjectType resourceType, + Collection resourceIds) { + if (CollectionUtils.isEmpty(resourceIds)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("resourceId", sb.entity().getResourceId(), SearchCriteria.Op.IN); + sb.and("resourceType", sb.entity().getResourceType(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("resourceId", resourceIds.toArray()); + sc.setParameters("resourceType", resourceType); + return listBy(sc); + } + + @Override + public List listByResourceTypeAndUuids(ResourceTag.ResourceObjectType resourceType, + Collection resourceUuids) { + if (CollectionUtils.isEmpty(resourceUuids)) { + return new ArrayList<>(); + } SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("uuid", resourceUuids.toArray()); sc.setParameters("resourceType", resourceType); - List resourceIcons = listBy(sc); + return listBy(sc); + } + + @Override + public List listResourceIcons(List resourceUuids, ResourceTag.ResourceObjectType resourceType) { + List resourceIcons = listByResourceTypeAndUuids(resourceType, resourceUuids); List iconResponses = new ArrayList<>(); for (ResourceIconVO resourceIcon : resourceIcons) { ResourceIconResponse response = new ResourceIconResponse(); diff --git a/engine/schema/src/main/java/com/cloud/storage/GuestOSCategoryVO.java b/engine/schema/src/main/java/com/cloud/storage/GuestOSCategoryVO.java index 36773e351e3..642705ffcbe 100644 --- a/engine/schema/src/main/java/com/cloud/storage/GuestOSCategoryVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/GuestOSCategoryVO.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.storage; +import java.util.Date; import java.util.UUID; import javax.persistence.Column; @@ -25,6 +26,8 @@ import javax.persistence.GenerationType; import javax.persistence.Id; import javax.persistence.Table; +import com.cloud.utils.db.GenericDao; + @Entity @Table(name = "guest_os_category") public class GuestOSCategoryVO implements GuestOsCategory { @@ -39,6 +42,26 @@ public class GuestOSCategoryVO implements GuestOsCategory { @Column(name = "uuid") String uuid = UUID.randomUUID().toString(); + @Column(name = "featured") + boolean featured; + + @Column(name = "sort_key") + private int sortKey; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + @Column(name = GenericDao.REMOVED_COLUMN) + private Date removed; + + public GuestOSCategoryVO() { + } + + public GuestOSCategoryVO(String name, boolean featured) { + this.name = name; + this.featured = featured; + } + @Override public long getId() { return id; @@ -59,7 +82,25 @@ public class GuestOSCategoryVO implements GuestOsCategory { return this.uuid; } - public void setUuid(String uuid) { - this.uuid = uuid; + @Override + public boolean isFeatured() { + return featured; + } + + public void setFeatured(Boolean featured) { + this.featured = featured; + } + + public void setSortKey(int key) { + sortKey = key; + } + + public int getSortKey() { + return sortKey; + } + + @Override + public Date getCreated() { + return created; } } diff --git a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java index 10d08601515..93f6a464019 100644 --- a/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java +++ b/engine/schema/src/main/java/com/cloud/storage/VMTemplateVO.java @@ -162,6 +162,9 @@ public class VMTemplateVO implements VirtualMachineTemplate { @Column(name = "deploy_as_is") private boolean deployAsIs; + @Column(name = "for_cks") + private boolean forCks; + @Column(name = "user_data_id") private Long userDataId; @@ -664,6 +667,14 @@ public class VMTemplateVO implements VirtualMachineTemplate { this.deployAsIs = deployAsIs; } + public boolean isForCks() { + return forCks; + } + + public void setForCks(boolean forCks) { + this.forCks = forCks; + } + @Override public Long getUserDataId() { return userDataId; diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java index 6fad6c5c47e..cc1c96aee52 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSCategoryDaoImpl.java @@ -27,7 +27,6 @@ import com.cloud.utils.db.GenericDaoBase; public class GuestOSCategoryDaoImpl extends GenericDaoBase implements GuestOSCategoryDao { protected GuestOSCategoryDaoImpl() { - } @Override diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java index 13cd398073a..1a2b098c40a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java @@ -16,14 +16,14 @@ // under the License. package com.cloud.storage.dao; +import java.util.List; +import java.util.Set; + import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSVO; import com.cloud.utils.Pair; import com.cloud.utils.db.GenericDao; -import java.util.List; -import java.util.Set; - public interface GuestOSDao extends GenericDao { GuestOSVO findOneByDisplayName(String displayName); @@ -36,4 +36,6 @@ public interface GuestOSDao extends GenericDao { List listByDisplayName(String displayName); Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay); + + List listIdsByCategoryId(final long categoryId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java index efcaa482a67..881be207c1a 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java @@ -25,19 +25,20 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import com.cloud.storage.GuestOS; -import com.cloud.utils.Pair; -import com.cloud.utils.db.DB; -import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; +import com.cloud.storage.GuestOS; import com.cloud.storage.GuestOSVO; +import com.cloud.utils.Pair; +import com.cloud.utils.db.DB; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.exception.CloudRuntimeException; @Component public class GuestOSDaoImpl extends GenericDaoBase implements GuestOSDao { @@ -152,4 +153,14 @@ public class GuestOSDaoImpl extends GenericDaoBase implements G return new Pair<>(result.first(), result.second()); } + @Override + public List listIdsByCategoryId(final long categoryId) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.selectFields(sb.entity().getId()); + sb.and("categoryId", sb.entity().getCategoryId(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("categoryId", categoryId); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index 0b40366a866..2835cf3cb3c 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -58,6 +58,8 @@ public interface VMTemplateDao extends GenericDao, StateDao< public List listInZoneByState(long dataCenterId, VirtualMachineTemplate.State... states); + public List listTemplateIsoByArchVnfAndZone(Long dataCenterId, CPU.CPUArch arch, Boolean isIso, Boolean isVnf); + public List listAllActive(); public List listByState(VirtualMachineTemplate.State... states); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 12c00a3209a..138677927e6 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -17,6 +17,7 @@ package com.cloud.storage.dao; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.Date; @@ -520,6 +521,48 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem return listBy(sc); } + @Override + public List listTemplateIsoByArchVnfAndZone(Long dataCenterId, CPU.CPUArch arch, Boolean isIso, + Boolean isVnf) { + GenericSearchBuilder sb = createSearchBuilder(Long.class); + sb.select(null, Func.DISTINCT, sb.entity().getGuestOSId()); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.IN); + sb.and("type", sb.entity().getTemplateType(), SearchCriteria.Op.IN); + sb.and("arch", sb.entity().getArch(), SearchCriteria.Op.EQ); + if (isIso != null) { + sb.and("isIso", sb.entity().getFormat(), isIso ? SearchCriteria.Op.EQ : SearchCriteria.Op.NEQ); + } + if (dataCenterId != null) { + SearchBuilder templateZoneSearch = _templateZoneDao.createSearchBuilder(); + templateZoneSearch.and("removed", templateZoneSearch.entity().getRemoved(), SearchCriteria.Op.NULL); + templateZoneSearch.and("zoneId", templateZoneSearch.entity().getZoneId(), SearchCriteria.Op.EQ); + sb.join("templateZoneSearch", templateZoneSearch, templateZoneSearch.entity().getTemplateId(), + sb.entity().getId(), JoinBuilder.JoinType.INNER); + templateZoneSearch.done(); + } + sb.done(); + SearchCriteria sc = sb.create(); + List types = new ArrayList<>(Arrays.asList(TemplateType.USER, TemplateType.BUILTIN, + TemplateType.PERHOST)); + if (isVnf == null) { + types.add(TemplateType.VNF); + } else if (isVnf) { + types = Collections.singletonList(TemplateType.VNF); + } + sc.setParameters("type", types.toArray()); + sc.setParameters("state", VirtualMachineTemplate.State.Active); + if (dataCenterId != null) { + sc.setJoinParameters("templateZoneSearch", "zoneId", dataCenterId); + } + if (arch != null) { + sc.setParameters("arch", arch); + } + if (isIso != null) { + sc.setParameters("isIso", ImageFormat.ISO); + } + return customSearch(sc, null); + } + @Override public List listAllActive() { SearchCriteria sc = ActiveTmpltSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java index d6dc85dbb9a..fd52782e57c 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42010to42100.java @@ -23,8 +23,12 @@ import com.cloud.utils.exception.CloudRuntimeException; import java.io.InputStream; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.cloudstack.framework.config.ConfigKey; @@ -59,6 +63,7 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr @Override public void performDataMigration(Connection conn) { + updateKubernetesClusterNodeVersions(conn); migrateConfigurationScopeToBitmask(conn); } @@ -88,6 +93,95 @@ public class Upgrade42010to42100 extends DbUpgradeAbstractImpl implements DbUpgr } } + private void updateKubernetesClusterNodeVersions(Connection conn) { + //get list of all non removed kubernetes clusters + try { + Map clusterAndVersion = getKubernetesClusterIdsAndVersion(conn); + updateKubernetesNodeVersions(conn, clusterAndVersion); + } catch (Exception e) { + String errMsg = "Failed to update kubernetes cluster nodes version"; + logger.error(errMsg); + throw new CloudRuntimeException(errMsg, e); + } + } + + private Map getKubernetesClusterIdsAndVersion(Connection conn) { + String listKubernetesClusters = "SELECT c.id, v.semantic_version FROM `cloud`.`kubernetes_cluster` c JOIN `cloud`.`kubernetes_supported_version` v ON (c.kubernetes_version_id = v.id) WHERE c.removed is NULL;"; + Map clusterAndVersion = new HashMap<>(); + try { + PreparedStatement pstmt = conn.prepareStatement(listKubernetesClusters); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + clusterAndVersion.put(rs.getLong(1), rs.getString(2)); + } + rs.close(); + pstmt.close(); + } catch (SQLException e) { + String errMsg = String.format("Failed to get all the kubernetes cluster ids due to: %s", e.getMessage()); + logger.error(errMsg); + throw new CloudRuntimeException(errMsg, e); + } + return clusterAndVersion; + } + + private void updateKubernetesNodeVersions(Connection conn, Map clusterAndVersion) { + List kubernetesClusterVmIds; + for (Map.Entry clusterVersionEntry : clusterAndVersion.entrySet()) { + try { + Long cksClusterId = clusterVersionEntry.getKey(); + String cksVersion = clusterVersionEntry.getValue(); + logger.debug(String.format("Adding CKS version %s to existing CKS cluster %s nodes", cksVersion, cksClusterId)); + kubernetesClusterVmIds = getKubernetesClusterVmMapIds(conn, cksClusterId); + updateKubernetesNodeVersion(conn, kubernetesClusterVmIds, cksClusterId, cksVersion); + } catch (Exception e) { + String errMsg = String.format("Failed to update the node version for kubernetes cluster nodes for the" + + " kubernetes cluster with id: %s," + + " due to: %s", clusterVersionEntry.getKey(), e.getMessage()); + logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } + } + + private List getKubernetesClusterVmMapIds(Connection conn, Long cksClusterId) { + List kubernetesClusterVmIds = new ArrayList<>(); + String getKubernetesClustersVmMap = "SELECT id FROM `cloud`.`kubernetes_cluster_vm_map` WHERE cluster_id = %s;"; + try { + PreparedStatement pstmt = conn.prepareStatement(String.format(getKubernetesClustersVmMap, cksClusterId)); + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + kubernetesClusterVmIds.add(rs.getLong(1)); + } + rs.close(); + pstmt.close(); + } catch (SQLException e) { + String errMsg = String.format("Failed to get the kubernetes cluster vm map IDs for kubernetes cluster with id: %s," + + " due to: %s", cksClusterId, e.getMessage()); + logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + return kubernetesClusterVmIds; + } + + private void updateKubernetesNodeVersion(Connection conn, List kubernetesClusterVmIds, Long cksClusterId, String cksVersion) { + String updateKubernetesNodeVersion = "UPDATE `cloud`.`kubernetes_cluster_vm_map` set kubernetes_node_version = ? WHERE id = ?;"; + for (Long nodeVmId : kubernetesClusterVmIds) { + try { + PreparedStatement pstmt = conn.prepareStatement(updateKubernetesNodeVersion); + pstmt.setString(1, cksVersion); + pstmt.setLong(2, nodeVmId); + pstmt.executeUpdate(); + pstmt.close(); + } catch (Exception e) { + String errMsg = String.format("Failed to update the node version for kubernetes cluster nodes for the" + + " kubernetes cluster with id: %s," + + " due to: %s", cksClusterId, e.getMessage()); + logger.error(errMsg, e); + throw new CloudRuntimeException(errMsg, e); + } + } + } + protected void migrateConfigurationScopeToBitmask(Connection conn) { String scopeDataType = DbUpgradeUtils.getTableColumnType(conn, "configuration", "scope"); logger.info("Data type of the column scope of table configuration is {}", scopeDataType); diff --git a/engine/schema/src/main/java/com/cloud/user/UserDataVO.java b/engine/schema/src/main/java/com/cloud/user/UserDataVO.java index a8e48ad22b1..e8864976069 100644 --- a/engine/schema/src/main/java/com/cloud/user/UserDataVO.java +++ b/engine/schema/src/main/java/com/cloud/user/UserDataVO.java @@ -65,6 +65,9 @@ public class UserDataVO implements UserData { @Column(name = GenericDao.REMOVED_COLUMN) private Date removed; + @Column(name = "for_cks") + private boolean forCks; + @Override public long getDomainId() { return domainId; @@ -105,6 +108,11 @@ public class UserDataVO implements UserData { return params; } + @Override + public boolean isForCks() { + return forCks; + } + public void setAccountId(long accountId) { this.accountId = accountId; } @@ -132,4 +140,6 @@ public class UserDataVO implements UserData { public Date getRemoved() { return removed; } + + public void setForCks(boolean forCks) { this.forCks = forCks; } } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 5a3c83f9b20..12432e80873 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -34,10 +34,47 @@ INSERT INTO `cloud`.`role_permissions` (uuid, role_id, rule, permission, sort_or SELECT uuid(), role_id, 'quotaCreditsList', permission, sort_order FROM `cloud`.`role_permissions` rp WHERE rp.rule = 'quotaStatement' -AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); + AND NOT EXISTS(SELECT 1 FROM cloud.role_permissions rp_ WHERE rp.role_id = rp_.role_id AND rp_.rule = 'quotaCreditsList'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host', 'last_mgmt_server_id', 'bigint unsigned DEFAULT NULL COMMENT "last management server this host is connected to" AFTER `mgmt_server_id`'); +----------------------------------------------------------- +-- CKS Enhancements: +----------------------------------------------------------- +-- Add for_cks column to the vm_template table +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vm_template','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the template can be used for CKS cluster deployment"'); + +-- Add support for different node types service offerings on CKS clusters +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_node_service_offering_id', 'bigint unsigned COMMENT "service offering ID for Control Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_node_service_offering_id', 'bigint unsigned COMMENT "service offering ID for Worker Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_node_service_offering_id', 'bigint unsigned COMMENT "service offering ID for etcd Nodes"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_node_count', 'bigint unsigned COMMENT "number of etcd nodes to be deployed for the Kubernetes cluster"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','control_node_template_id', 'bigint unsigned COMMENT "template id to be used for Control Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','worker_node_template_id', 'bigint unsigned COMMENT "template id to be used for Worker Node(s)"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','etcd_node_template_id', 'bigint unsigned COMMENT "template id to be used for etcd Nodes"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','cni_config_id', 'bigint unsigned COMMENT "user data id representing the associated cni configuration"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster','cni_config_details', 'varchar(4096) DEFAULT NULL COMMENT "user data details representing the values required for the cni configuration associated"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','etcd_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the VM is an etcd node"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','external_node', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node was imported into the Kubernetes cluster"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','manual_upgrade', 'tinyint(1) unsigned NOT NULL DEFAULT 0 COMMENT "indicates if the node is marked for manual upgrade and excluded from the Kubernetes cluster upgrade operation"'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.kubernetes_cluster_vm_map','kubernetes_node_version', 'varchar(40) COMMENT "version of k8s the cluster node is on"'); + +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_node_service_offering_id` FOREIGN KEY `fk_cluster__control_node_service_offering_id`(`control_node_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_node_service_offering_id` FOREIGN KEY `fk_cluster__worker_node_service_offering_id`(`worker_node_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_node_service_offering_id` FOREIGN KEY `fk_cluster__etcd_node_service_offering_id`(`etcd_node_service_offering_id`) REFERENCES `service_offering`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__control_node_template_id` FOREIGN KEY `fk_cluster__control_node_template_id`(`control_node_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__worker_node_template_id` FOREIGN KEY `fk_cluster__worker_node_template_id`(`worker_node_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; +ALTER TABLE `cloud`.`kubernetes_cluster` ADD CONSTRAINT `fk_cluster__etcd_node_template_id` FOREIGN KEY `fk_cluster__etcd_node_template_id`(`etcd_node_template_id`) REFERENCES `vm_template`(`id`) ON DELETE CASCADE; + +-- Add for_cks column to the user_data table to represent CNI Configuration stored as userdata +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_data','for_cks', 'int(1) unsigned DEFAULT "0" COMMENT "if true, the user data represent CNI configuration meant for CKS use only"'); + +-- Add use VR IP as resolver option on VPC +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc','use_router_ip_resolver', 'tinyint(1) DEFAULT 0 COMMENT "use router ip as resolver instead of dns options"'); +----------------------------------------------------------- +-- END - CKS Enhancements +----------------------------------------------------------- + -- Add table for reconcile commands CREATE TABLE IF NOT EXISTS `cloud`.`reconcile_commands` ( `id` bigint unsigned NOT NULL UNIQUE AUTO_INCREMENT, @@ -81,9 +118,95 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.cluster', 'storage_access_groups', ' CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.host_pod_ref', 'storage_access_groups', 'varchar(255) DEFAULT NULL COMMENT "storage access groups for the hosts in the pod"'); CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.data_center', 'storage_access_groups', 'varchar(255) DEFAULT NULL COMMENT "storage access groups for the hosts in the zone"'); +-- Add featured, sort_key, created, removed columns for guest_os_category +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os_category', 'featured', 'tinyint(1) NOT NULL DEFAULT 0 COMMENT "whether the category is featured or not" AFTER `uuid`'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os_category', 'sort_key', 'int NOT NULL DEFAULT 0 COMMENT "sort key used for customising sort method" AFTER `featured`'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os_category', 'created', 'datetime COMMENT "date on which the category was created" AFTER `sort_key`'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.guest_os_category', 'removed', 'datetime COMMENT "date removed if not null" AFTER `created`'); + +-- Begin: Changes for Guest OS category cleanup +-- Add new OS categories if not present +DROP PROCEDURE IF EXISTS `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`; +CREATE PROCEDURE `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`(IN os_name VARCHAR(255)) +BEGIN + IF NOT EXISTS ((SELECT 1 FROM `cloud`.`guest_os_category` WHERE name = os_name)) + THEN + INSERT INTO `cloud`.`guest_os_category` (name, uuid) + VALUES (os_name, UUID()) +; END IF +; END; + +CALL `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`('Fedora'); +CALL `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`('Rocky Linux'); +CALL `cloud`.`INSERT_CATEGORY_IF_NOT_EXIST`('AlmaLinux'); + +-- Move existing guest OS to new categories +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`; +CREATE PROCEDURE `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`(IN category_name VARCHAR(255), IN os_name VARCHAR(255)) +BEGIN + DECLARE category_id BIGINT +; SELECT `id` INTO category_id + FROM `cloud`.`guest_os_category` + WHERE `name` = category_name + LIMIT 1 +; IF category_id IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Category not found' +; END IF +; UPDATE `cloud`.`guest_os` + SET `category_id` = category_id + WHERE `display_name` LIKE CONCAT('%', os_name, '%') +; END; +CALL `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`('Rocky Linux', 'Rocky Linux'); +CALL `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`('AlmaLinux', 'AlmaLinux'); +CALL `cloud`.`UPDATE_CATEGORY_FOR_GUEST_OSES`('Fedora', 'Fedora'); + +-- Move existing guest OS whose category will be deleted to Other category +DROP PROCEDURE IF EXISTS `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`; +CREATE PROCEDURE `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`(IN to_category_name VARCHAR(255), IN from_category_name VARCHAR(255)) +BEGIN + DECLARE done INT DEFAULT 0 +; DECLARE to_category_id BIGINT +; SELECT id INTO to_category_id + FROM `cloud`.`guest_os_category` + WHERE `name` = to_category_name + LIMIT 1 +; IF to_category_id IS NULL THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'ToCategory not found' +; END IF +; UPDATE `cloud`.`guest_os` + SET `category_id` = to_category_id + WHERE `category_id` = (SELECT `id` FROM `cloud`.`guest_os_category` WHERE `name` = from_category_name) +; UPDATE `cloud`.`guest_os_category` SET `removed`=now() WHERE `name` = from_category_name +; END; +CALL `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`('Other', 'Novel'); +CALL `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`('Other', 'None'); +CALL `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`('Other', 'Unix'); +CALL `cloud`.`UPDATE_NEW_AND_DELETE_OLD_CATEGORY_FOR_GUEST_OS`('Other', 'Mac'); + +-- Update featured for existing guest OS categories +UPDATE `cloud`.`guest_os_category` SET featured = 1; + +-- Update sort order for all guest OS categories +UPDATE `cloud`.`guest_os_category` +SET `sort_key` = CASE + WHEN `name` = 'Ubuntu' THEN 1 + WHEN `name` = 'Debian' THEN 2 + WHEN `name` = 'Fedora' THEN 3 + WHEN `name` = 'CentOS' THEN 4 + WHEN `name` = 'Rocky Linux' THEN 5 + WHEN `name` = 'AlmaLinux' THEN 6 + WHEN `name` = 'Oracle' THEN 7 + WHEN `name` = 'RedHat' THEN 8 + WHEN `name` = 'SUSE' THEN 9 + WHEN `name` = 'Windows' THEN 10 + WHEN `name` = 'Other' THEN 11 + ELSE `sort_key` +END; +-- End: Changes for Guest OS category cleanup + -- Disk controller mappings CREATE TABLE IF NOT EXISTS `cloud`.`disk_controller_mapping` ( - `id` bigint(20) unsigned NOT NULL auto_increment, + `id` bigint(20) unsigned NOT NULL auto_increment, `uuid` varchar(255) UNIQUE NOT NULL, `name` varchar(255) NOT NULL, `controller_reference` varchar(255) NOT NULL, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql index 339e43860d8..6bfcdaddbcc 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.template_view.sql @@ -41,6 +41,7 @@ SELECT `vm_template`.`guest_os_id` AS `guest_os_id`, `guest_os`.`uuid` AS `guest_os_uuid`, `guest_os`.`display_name` AS `guest_os_name`, + `guest_os`.`category_id` AS `guest_os_category_id`, `vm_template`.`bootable` AS `bootable`, `vm_template`.`prepopulate` AS `prepopulate`, `vm_template`.`cross_zones` AS `cross_zones`, @@ -100,6 +101,7 @@ SELECT IFNULL(`data_center`.`id`, 0)) AS `temp_zone_pair`, `vm_template`.`direct_download` AS `direct_download`, `vm_template`.`deploy_as_is` AS `deploy_as_is`, + `vm_template`.`for_cks` AS `for_cks`, `user_data`.`id` AS `user_data_id`, `user_data`.`uuid` AS `user_data_uuid`, `user_data`.`name` AS `user_data_name`, diff --git a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java index df0b36ebdbf..96f60547f50 100644 --- a/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java +++ b/engine/schema/src/test/java/com/cloud/storage/dao/VMTemplateDaoImplTest.java @@ -21,7 +21,11 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.util.ArrayList; @@ -44,8 +48,11 @@ import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.Storage; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VMTemplateZoneVO; import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericSearchBuilder; +import com.cloud.utils.db.JoinBuilder; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; @@ -55,6 +62,9 @@ public class VMTemplateDaoImplTest { @Mock HostDao hostDao; + @Mock + VMTemplateZoneDao templateZoneDao; + @Spy @InjectMocks VMTemplateDaoImpl templateDao = new VMTemplateDaoImpl(); @@ -186,4 +196,107 @@ public class VMTemplateDaoImplTest { VMTemplateVO result = templateDao.findLatestTemplateByTypeAndHypervisorAndArch(hypervisorType, arch, type); assertNull(result); } + + private void mockTemplateZoneJoin() { + VMTemplateZoneVO templateZoneVO = mock(VMTemplateZoneVO.class); + SearchBuilder templateZoneVOSearchBuilder = mock(SearchBuilder.class); + when(templateZoneVOSearchBuilder.entity()).thenReturn(templateZoneVO); + when(templateZoneDao.createSearchBuilder()).thenReturn(templateZoneVOSearchBuilder); + } + + @Test + public void testListTemplateIsoByArchAndZone_WithDataCenterId() { + Long dataCenterId = 1L; + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + Boolean isIso = true; + VMTemplateVO templateVO = mock(VMTemplateVO.class); + GenericSearchBuilder searchBuilder = mock(GenericSearchBuilder.class); + when(searchBuilder.entity()).thenReturn(templateVO); + SearchCriteriasearchCriteria = mock(SearchCriteria.class); + when(templateDao.createSearchBuilder(Long.class)).thenReturn(searchBuilder); + when(searchBuilder.create()).thenReturn(searchCriteria); + mockTemplateZoneJoin(); + doReturn(new ArrayList<>()).when(templateDao).customSearch(searchCriteria, null); + List result = templateDao.listTemplateIsoByArchVnfAndZone(dataCenterId, arch, isIso, false); + assertNotNull(result); + verify(searchBuilder, times(1)).select(null, SearchCriteria.Func.DISTINCT, templateVO.getGuestOSId()); + verify(searchBuilder, times(1)).and(eq("state"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("type"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("arch"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, times(1)).and(eq("isIso"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, times(1)).join(eq("templateZoneSearch"), any(), any(), any(), eq(JoinBuilder.JoinType.INNER)); + verify(templateDao, times(1)).customSearch(searchCriteria, null); + } + + @Test + public void testListTemplateIsoByArchAndZone_WithoutDataCenterId() { + Long dataCenterId = null; + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + Boolean isIso = false; + VMTemplateVO templateVO = mock(VMTemplateVO.class); + GenericSearchBuilder searchBuilder = mock(GenericSearchBuilder.class); + when(searchBuilder.entity()).thenReturn(templateVO); + SearchCriteriasearchCriteria = mock(SearchCriteria.class); + when(templateDao.createSearchBuilder(Long.class)).thenReturn(searchBuilder); + when(searchBuilder.create()).thenReturn(searchCriteria); + doReturn(new ArrayList<>()).when(templateDao).customSearch(searchCriteria, null); + List result = templateDao.listTemplateIsoByArchVnfAndZone(dataCenterId, arch, isIso, false); + assertNotNull(result); + verify(searchBuilder, times(1)).select(null, SearchCriteria.Func.DISTINCT, templateVO.getGuestOSId()); + verify(searchBuilder, times(1)).and(eq("state"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("type"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("arch"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, times(1)).and(eq("isIso"), any(), eq(SearchCriteria.Op.NEQ)); + verify(searchBuilder, never()).join(eq("templateZoneSearch"), any(), any(), any(), eq(JoinBuilder.JoinType.INNER)); + verify(templateDao, times(1)).customSearch(searchCriteria, null); + } + + @Test + public void testListTemplateIsoByArchAndZone_WithoutArch() { + Long dataCenterId = 1L; + CPU.CPUArch arch = null; + Boolean isIso = true; + VMTemplateVO templateVO = mock(VMTemplateVO.class); + GenericSearchBuilder searchBuilder = mock(GenericSearchBuilder.class); + when(searchBuilder.entity()).thenReturn(templateVO); + SearchCriteriasearchCriteria = mock(SearchCriteria.class); + when(templateDao.createSearchBuilder(Long.class)).thenReturn(searchBuilder); + when(searchBuilder.create()).thenReturn(searchCriteria); + mockTemplateZoneJoin(); + doReturn(new ArrayList<>()).when(templateDao).customSearch(searchCriteria, null); + List result = templateDao.listTemplateIsoByArchVnfAndZone(dataCenterId, arch, isIso, false); + assertNotNull(result); + verify(searchBuilder, times(1)).select(null, SearchCriteria.Func.DISTINCT, templateVO.getGuestOSId()); + verify(searchBuilder, times(1)).and(eq("state"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("type"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("arch"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, times(1)).and(eq("isIso"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, times(1)).join(eq("templateZoneSearch"), any(), any(), any(), eq(JoinBuilder.JoinType.INNER)); + verify(templateDao, times(1)).customSearch(searchCriteria, null); + } + + @Test + public void testListTemplateIsoByArchAndZone_WithoutIsIso() { + Long dataCenterId = 1L; + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + Boolean isIso = null; + VMTemplateVO templateVO = mock(VMTemplateVO.class); + GenericSearchBuilder searchBuilder = mock(GenericSearchBuilder.class); + when(searchBuilder.entity()).thenReturn(templateVO); + SearchCriteriasearchCriteria = mock(SearchCriteria.class); + when(templateDao.createSearchBuilder(Long.class)).thenReturn(searchBuilder); + when(searchBuilder.create()).thenReturn(searchCriteria); + mockTemplateZoneJoin(); + doReturn(new ArrayList<>()).when(templateDao).customSearch(searchCriteria, null); + List result = templateDao.listTemplateIsoByArchVnfAndZone(dataCenterId, arch, isIso, false); + assertNotNull(result); + verify(searchBuilder, times(1)).select(null, SearchCriteria.Func.DISTINCT, templateVO.getGuestOSId()); + verify(searchBuilder, times(1)).and(eq("state"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("type"), any(), eq(SearchCriteria.Op.IN)); + verify(searchBuilder, times(1)).and(eq("arch"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, never()).and(eq("isIso"), any(), eq(SearchCriteria.Op.NEQ)); + verify(searchBuilder, never()).and(eq("isIso"), any(), eq(SearchCriteria.Op.EQ)); + verify(searchBuilder, times(1)).join(eq("templateZoneSearch"), any(), any(), any(), eq(JoinBuilder.JoinType.INNER)); + verify(templateDao, times(1)).customSearch(searchCriteria, null); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java index 035790f0716..16908f6aaac 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/dao/Upgrade42010to42100Test.java @@ -16,10 +16,12 @@ // under the License. package com.cloud.upgrade.dao; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; import java.sql.Connection; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import org.junit.Test; @@ -64,6 +66,12 @@ public class Upgrade42010to42100Test { " ELSE 0" + " END WHERE scope IS NOT NULL;"; when(txn.prepareAutoCloseStatement(sql)).thenReturn(pstmt); + + PreparedStatement preparedStatement = Mockito.mock(PreparedStatement.class); + ResultSet resultSet = Mockito.mock(ResultSet.class); + Mockito.when(resultSet.next()).thenReturn(false); + Mockito.when(preparedStatement.executeQuery()).thenReturn(resultSet); + Mockito.when(conn.prepareStatement(anyString())).thenReturn(preparedStatement); upgrade.performDataMigration(conn); Mockito.verify(pstmt, Mockito.times(1)).executeUpdate(); diff --git a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java index a3b7d0c9ecc..0dbe4fd7246 100644 --- a/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/main/java/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -433,6 +433,11 @@ public class TemplateObject implements TemplateInfo { return this.imageVO.isDeployAsIs(); } + @Override + public boolean isForCks() { + return imageVO.isForCks(); + } + public void setInstallPath(String installPath) { this.installPath = installPath; } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java index 25e4608e58f..e7666965661 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ClusterScopeStoragePoolAllocator.java @@ -100,7 +100,7 @@ public class ClusterScopeStoragePoolAllocator extends AbstractStoragePoolAllocat } StoragePool storagePool = (StoragePool)dataStoreMgr.getPrimaryDataStore(pool.getId()); if (filter(avoid, storagePool, dskCh, plan)) { - logger.debug(String.format("Found suitable cluster storage pool [%s] to allocate disk [%s] to it, adding to list.", pool, dskCh)); + logger.debug("Found suitable cluster storage pool [{}] to allocate disk [{}] to it, adding to list.", pool, dskCh); suitablePools.add(storagePool); } else { logger.debug(String.format("Adding storage pool [%s] to avoid set during allocation of disk [%s].", pool, dskCh)); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java index 13b5f8e4814..bdf531e147b 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/ZoneWideStoragePoolAllocator.java @@ -96,7 +96,7 @@ public class ZoneWideStoragePoolAllocator extends AbstractStoragePoolAllocator { } StoragePool storagePool = (StoragePool)this.dataStoreMgr.getPrimaryDataStore(storage.getId()); if (filter(avoid, storagePool, dskCh, plan)) { - logger.debug(String.format("Found suitable zone wide storage pool [%s] to allocate disk [%s] to it, adding to list.", storagePool, dskCh)); + logger.debug("Found suitable zone wide storage pool [{}] to allocate disk [{}] to it, adding to list.", storagePool, dskCh); suitablePools.add(storagePool); } else { if (canAddStoragePoolToAvoidSet(storage)) { diff --git a/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java b/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java index b136b8e842b..780a09b883e 100644 --- a/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java +++ b/framework/agent-lb/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLB.java @@ -70,9 +70,11 @@ public interface IndirectAgentLB { */ Long getLBPreferredHostCheckInterval(Long clusterId); - void propagateMSListToAgents(); + void propagateMSListToAgents(boolean triggerHostLB); - boolean haveAgentBasedHosts(long msId); + void propagateMSListToAgentsInCluster(Long clusterId); - boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorithm, long timeoutDurationInMs); + boolean haveAgentBasedHosts(long msId, boolean excludeHostsInMaintenance); + + boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorithm, long timeoutDurationInMs, boolean excludeHostsInMaintenance); } diff --git a/framework/db/src/main/java/com/cloud/dao/EntityManagerImpl.java b/framework/db/src/main/java/com/cloud/dao/EntityManagerImpl.java index beee1b608d0..b148a8787e7 100644 --- a/framework/db/src/main/java/com/cloud/dao/EntityManagerImpl.java +++ b/framework/db/src/main/java/com/cloud/dao/EntityManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.dao; import java.io.Serializable; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -56,6 +57,13 @@ public class EntityManagerImpl extends ManagerBase implements EntityManager { return dao.findByUuid(uuid); } + @Override + public List listByUuids(Class entityType, Collection uuids) { + // Finds and returns a unique VO using uuid, null if entity not found in db + GenericDao dao = (GenericDao)GenericDaoBase.getDao(entityType); + return (List)dao.listByUuids(uuids); + } + @Override public T findByUuidIncludingRemoved(Class entityType, String uuid) { // Finds and returns a unique VO using uuid, null if entity not found in db diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java index 44c312ea9d8..89bfeb81861 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java @@ -17,6 +17,8 @@ package com.cloud.utils.db; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -56,6 +58,10 @@ public interface GenericDao { // Finds one unique VO using uuid T findByUuid(String uuid); + default List listByUuids(Collection uuids) { + return new ArrayList<>(); + } + // Finds one unique VO using uuid including removed entities T findByUuidIncludingRemoved(String uuid); diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index 301803aab9b..560cb4494ee 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -1006,6 +1006,17 @@ public abstract class GenericDaoBase extends Compone return findOneBy(sc); } + @Override + @DB() + public List listByUuids(final Collection uuids) { + if (org.apache.commons.collections.CollectionUtils.isEmpty(uuids)) { + return Collections.emptyList(); + } + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("uuid", SearchCriteria.Op.IN, uuids.toArray()); + return listBy(sc); + } + @Override @DB() public T findByUuidIncludingRemoved(final String uuid) { diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 22fede6fb85..2c6898cac7c 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -248,6 +248,7 @@ cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d @@ -273,7 +274,7 @@ wget https://github.com/apache/cloudstack-cloudmonkey/releases/download/$CMK_REL chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup - +cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/ @@ -308,6 +309,11 @@ touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml + # SystemVM template mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm @@ -608,6 +614,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz %attr(0755,root,root) %{_bindir}/%{name}-sysvmadm %attr(0755,root,root) %{_bindir}/%{name}-setup-encryption %attr(0755,root,root) %{_bindir}/cmk +%{_datadir}/%{name}-management/cks/conf/*.yml %{_datadir}/%{name}-management/setup/*.sql %{_datadir}/%{name}-management/setup/*.sh %{_datadir}/%{name}-management/setup/server-setup.xml diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index a632fd5adfd..3288e2d6be1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -17,6 +17,8 @@ package com.cloud.hypervisor.kvm.resource; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; +import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; +import static com.cloud.host.Host.HOST_VIRTV2V_VERSION; import static com.cloud.host.Host.HOST_VOLUME_ENCRYPTION; import static org.apache.cloudstack.utils.linux.KVMHostInfo.isHostS390x; @@ -3908,7 +3910,14 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setIqn(getIqn()); cmd.getHostDetails().put(HOST_VOLUME_ENCRYPTION, String.valueOf(hostSupportsVolumeEncryption())); cmd.setHostTags(getHostTags()); - cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(hostSupportsInstanceConversion())); + boolean instanceConversionSupported = hostSupportsInstanceConversion(); + cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + if (instanceConversionSupported) { + cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); + } + if (hostSupportsOvfExport()) { + cmd.getHostDetails().put(HOST_OVFTOOL_VERSION, getHostOvfToolVersion()); + } HealthCheckResult healthCheckResult = getHostHealthCheckResult(); if (healthCheckResult != HealthCheckResult.IGNORE) { cmd.setHostHealthCheckResult(healthCheckResult == HealthCheckResult.SUCCESS); @@ -5616,8 +5625,24 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + public String getHostVirtV2vVersion() { + if (!hostSupportsInstanceConversion()) { + return ""; + } + String cmd = String.format("%s | awk '{print $2}'", INSTANCE_CONVERSION_SUPPORTED_CHECK_CMD); + String version = Script.runSimpleBashScript(cmd); + return StringUtils.isNotBlank(version) ? version.split(",")[0] : ""; + } + + public String getHostOvfToolVersion() { + if (!hostSupportsOvfExport()) { + return ""; + } + return Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + } + public boolean ovfExportToolSupportsParallelThreads() { - String ovfExportToolVersion = Script.runSimpleBashScript(OVF_EXPORT_TOOl_GET_VERSION_CMD); + String ovfExportToolVersion = getHostOvfToolVersion(); if (StringUtils.isBlank(ovfExportToolVersion)) { return false; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index 8f23e79e4a3..485254c6bb9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -47,6 +47,14 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper info = qemu.info(srcFile); + Long virtualSize = Long.parseLong(info.get(QemuImg.VIRTUAL_SIZE)); + KVMPhysicalDisk destDisk = new KVMPhysicalDisk(destPool.getSourceDir() + "/" + templateUuid, templateUuid, destPool); + destDisk.setFormat(PhysicalDiskFormat.RAW); + destDisk.setSize(virtualSize); + destDisk.setVirtualSize(virtualSize); + QemuImgFile destFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(destPool, destDisk.getPath())); + destFile.setFormat(PhysicalDiskFormat.RAW); + qemu.convert(srcFile, destFile); + } catch (LibvirtException | QemuImgException e) { + String err = String.format("Error creating template from direct download file on pool %s: %s", destPool.getUuid(), e.getMessage()); + logger.error(err, e); + throw new CloudRuntimeException(err, e); + } + } + public StorageVol getVolume(StoragePool pool, String volName) { StorageVol vol = null; diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index 751a16a1554..608ad3c69cb 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import org.apache.cloudstack.storage.formatinspector.Qcow2Inspector; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; @@ -53,6 +54,8 @@ public class QemuImg { public static final long QEMU_2_10 = 2010000; public static final long QEMU_5_10 = 5010000; + public static final int MIN_BITMAP_VERSION = 3; + /* The qemu-img binary. We expect this to be in $PATH */ public String _qemuImgPath = "qemu-img"; private String cloudQemuImgPath = "cloud-qemu-img"; @@ -466,7 +469,7 @@ public class QemuImg { script.add(srcFile.getFileName()); } - if (this.version >= QEMU_5_10 && keepBitmaps) { + if (this.version >= QEMU_5_10 && keepBitmaps && Qcow2Inspector.validateQcow2Version(srcFile.getFileName(), MIN_BITMAP_VERSION)) { script.add("--bitmaps"); } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java index 2ddf1bd1851..abf46b2771c 100755 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixResizeVolumeCommandWrapper.java @@ -49,9 +49,12 @@ public final class CitrixResizeVolumeCommandWrapper extends CommandWrapper= newSize) { - logger.info("No need to resize volume: " + volId +", current size " + toHumanReadableSize(command.getCurrentSize()) + " is same as new size " + toHumanReadableSize(newSize)); + if (command.getCurrentSize() == newSize) { + logger.info("No need to resize volume [{}], current size [{}] is same as new size [{}].", volId, toHumanReadableSize(command.getCurrentSize()), toHumanReadableSize(newSize)); return new ResizeVolumeAnswer(command, true, "success", newSize); + } else if (command.getCurrentSize() > newSize) { + logger.error("XenServer does not support volume shrink. Volume [{}] current size [{}] is smaller than new size [{}]", volId, toHumanReadableSize(command.getCurrentSize()), toHumanReadableSize(newSize)); + return new ResizeVolumeAnswer(command, false, "operation not supported"); } if (command.isManaged()) { resizeSr(conn, command); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java index a947e4273be..486a093e4ad 100755 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterEventTypes.java @@ -23,4 +23,6 @@ public class KubernetesClusterEventTypes { public static final String EVENT_KUBERNETES_CLUSTER_STOP = "KUBERNETES.CLUSTER.STOP"; public static final String EVENT_KUBERNETES_CLUSTER_SCALE = "KUBERNETES.CLUSTER.SCALE"; public static final String EVENT_KUBERNETES_CLUSTER_UPGRADE = "KUBERNETES.CLUSTER.UPGRADE"; + public static final String EVENT_KUBERNETES_CLUSTER_NODES_ADD = "KUBERNETES.CLUSTER.NODES.ADD"; + public static final String EVENT_KUBERNETES_CLUSTER_NODES_REMOVE = "KUBERNETES.CLUSTER.NODES.REMOVE"; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java index f551344ec0e..f242ded5593 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java @@ -16,13 +16,19 @@ // under the License. package com.cloud.kubernetes.cluster; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; import static com.cloud.vm.UserVmManager.AllowUserExpungeRecoverVm; +import java.lang.reflect.InvocationTargetException; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.EnumSet; import java.util.HashMap; @@ -41,8 +47,28 @@ import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.bgp.BGPService; +import com.cloud.dc.DedicatedResourceVO; +import com.cloud.dc.dao.DedicatedResourceDao; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.host.Host; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterRemoveWorker; +import com.cloud.network.dao.NsxProviderDao; +import com.cloud.network.element.NsxProviderVO; +import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterAddWorker; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.template.TemplateApiService; +import com.cloud.user.UserDataVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDataDao; import com.cloud.uservm.UserVm; +import com.cloud.vm.NicVO; import com.cloud.vm.UserVmService; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.dao.UserVmDetailsDao; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RolePermissionEntity; @@ -50,13 +76,18 @@ import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.Rule; import org.apache.cloudstack.acl.SecurityChecker; +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiConstants.VMDetails; +import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.ResponseObject.ResponseView; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd; import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd; import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; @@ -70,6 +101,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernete import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; @@ -89,6 +121,7 @@ import org.apache.cloudstack.api.command.user.network.ListNetworksCmd; import org.apache.cloudstack.api.command.user.vm.ListVMsCmd; import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse; import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.KubernetesUserVmResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.RemoveVirtualMachinesFromKubernetesClusterResponse; import org.apache.cloudstack.api.response.UserVmResponse; @@ -97,9 +130,11 @@ import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.commons.beanutils.BeanUtils; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -126,7 +161,6 @@ import com.cloud.exception.InsufficientServerCapacityException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; import com.cloud.exception.ResourceAllocationException; -import com.cloud.host.Host.Type; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; @@ -188,7 +222,6 @@ import com.cloud.user.SSHKeyPairVO; import com.cloud.user.User; import com.cloud.user.UserAccount; import com.cloud.user.UserVO; -import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.SSHKeyPairDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; @@ -251,6 +284,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); + protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER.name(), CONTROL.name(), ETCD.name()); + ScheduledExecutorService _gcExecutor; ScheduledExecutorService _stateScanner; @@ -273,18 +308,28 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject protected HostDao hostDao; @Inject + protected AffinityGroupDao affinityGroupDao; + @Inject protected ServiceOfferingDao serviceOfferingDao; @Inject + protected UserDataDao userDataDao; + @Inject protected VMTemplateDao templateDao; @Inject protected TemplateJoinDao templateJoinDao; @Inject + protected DedicatedResourceDao dedicatedResourceDao; + @Inject + protected AccountDao accountDao; + @Inject protected AccountService accountService; @Inject protected AccountManager accountManager; @Inject protected UserDao userDao; @Inject + protected UserVmDetailsDao userVmDetailsDao; + @Inject protected VMInstanceDao vmInstanceDao; @Inject protected UserVmJoinDao userVmJoinDao; @@ -321,11 +366,19 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Inject public NetworkHelper networkHelper; @Inject + private NsxProviderDao nsxProviderDao; + @Inject + private NicDao nicDao; + @Inject private UserVmService userVmService; @Inject + private TemplateApiService templateService; + @Inject + private PortForwardingRulesDao pfRuleDao; + @Inject RoutedIpv4Manager routedIpv4Manager; @Inject - public AccountDao accountDao; + private BGPService bgpService; @Inject public ProjectManager projectManager; @Inject @@ -432,16 +485,32 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return null; } - public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType) { + public VMTemplateVO getKubernetesServiceTemplate(DataCenter dataCenter, Hypervisor.HypervisorType hypervisorType, Map templateNodeTypeMap, KubernetesClusterNodeType nodeType) { VMTemplateVO template = templateDao.findSystemVMReadyTemplate(dataCenter.getId(), hypervisorType); if (DataCenter.Type.Edge.equals(dataCenter.getType()) && template != null && !template.isDirectDownload()) { logger.debug(String.format("Template %s can not be used for edge zone %s", template, dataCenter)); template = templateDao.findRoutingTemplate(hypervisorType, networkHelper.getHypervisorRouterTemplateConfigMap().get(hypervisorType).valueIn(dataCenter.getId())); } - if (template == null) { - throw new CloudRuntimeException("Not able to find the System or Routing template in ready state for the zone " + dataCenter.getUuid()); + switch (nodeType) { + case CONTROL: + case ETCD: + case WORKER: + VMTemplateVO nodeTemplate = Objects.nonNull(templateNodeTypeMap) ? templateDao.findById(templateNodeTypeMap.getOrDefault(nodeType.name(), 0L)) : template; + template = Objects.nonNull(nodeTemplate) ? nodeTemplate : template; + if (Objects.isNull(template)) { + throwDefaultCksTemplateNotFound(dataCenter.getUuid()); + } + return template; + default: + if (Objects.isNull(template)) { + throwDefaultCksTemplateNotFound(dataCenter.getUuid()); + } + return template; } - return template; + } + + public void throwDefaultCksTemplateNotFound(String datacenterId) { + throw new CloudRuntimeException("Not able to find the System or Routing template in ready state for the zone " + datacenterId); } protected void validateIsolatedNetworkIpRules(long ipId, FirewallRule.Purpose purpose, Network network, int clusterTotalNodeCount) { @@ -516,7 +585,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne validateIsolatedNetwork(network, clusterTotalNodeCount); } - private boolean validateServiceOffering(final ServiceOffering serviceOffering, final KubernetesSupportedVersion version) { + protected void validateServiceOffering(final ServiceOffering serviceOffering, final KubernetesSupportedVersion version) throws InvalidParameterValueException { if (serviceOffering.isDynamic()) { throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for creating clusters, service offering ID: %s", serviceOffering.getUuid())); } @@ -529,7 +598,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (serviceOffering.getRamSize() < version.getMinimumRamSize()) { throw new InvalidParameterValueException(String.format("Kubernetes cluster cannot be created with service offering ID: %s, associated Kubernetes version ID: %s needs minimum %d MB RAM", serviceOffering.getUuid(), version.getUuid(), version.getMinimumRamSize())); } - return true; } private void validateDockerRegistryParams(final String dockerRegistryUserName, @@ -557,16 +625,46 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { + public Long getExplicitAffinityGroup(Long domainId) { + AffinityGroupVO groupVO = affinityGroupDao.findDomainLevelGroupByType(domainId, "ExplicitDedication"); + if (Objects.nonNull(groupVO)) { + return groupVO.getId(); + } + return null; + } + + private DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering, + final Long domainId, final Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; - List hosts = resourceManager.listAllHostsInOneZoneByType(Type.Routing, zone.getId()); + boolean useDedicatedHosts = false; + Long group = getExplicitAffinityGroup(domainId); + List hosts = new ArrayList<>(); + if (Objects.nonNull(group)) { + List dedicatedHosts = new ArrayList<>(); + if (Objects.nonNull(accountId)) { + dedicatedHosts = dedicatedResourceDao.listByAccountId(accountId); + } else if (Objects.nonNull(domainId)) { + dedicatedHosts = dedicatedResourceDao.listByDomainId(domainId); + } + for (DedicatedResourceVO dedicatedHost : dedicatedHosts) { + hosts.add(hostDao.findById(dedicatedHost.getHostId())); + useDedicatedHosts = true; + } + } + if (hosts.isEmpty()) { + hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId()); + } + if (hypervisorType != null) { + hosts = hosts.stream().filter(x -> x.getHypervisorType() == hypervisorType).collect(Collectors.toList()); + } final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); for (HostVO h : hosts) { hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); } boolean suitable_host_found = false; Cluster planCluster = null; + HostVO suitableHost = null; for (int i = 1; i <= nodesCount; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { @@ -590,6 +688,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logger.debug("Found host {} to have enough capacity, CPU={} RAM={}", hostVO, cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved)); hostEntry.setValue(new Pair(hostVO, reserved)); suitable_host_found = true; + suitableHost = hostVO; planCluster = cluster; break; } @@ -605,6 +704,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (logger.isInfoEnabled()) { logger.info("Suitable hosts found in datacenter: {}, creating deployment destination", zone); } + if (useDedicatedHosts) { + planCluster = clusterDao.findById(suitableHost.getClusterId()); + return new DeployDestination(zone, null, planCluster, suitableHost); + } return new DeployDestination(zone, null, planCluster, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%s) with offering: %s", @@ -613,6 +716,33 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } + protected void setNodeTypeServiceOfferingResponse(KubernetesClusterResponse response, + KubernetesClusterNodeType nodeType, + Long offeringId) { + if (offeringId == null) { + return; + } + ServiceOfferingVO offering = serviceOfferingDao.findById(offeringId); + if (offering != null) { + setServiceOfferingResponseForNodeType(response, offering, nodeType); + } + } + + protected void setServiceOfferingResponseForNodeType(KubernetesClusterResponse response, + ServiceOfferingVO offering, + KubernetesClusterNodeType nodeType) { + if (CONTROL == nodeType) { + response.setControlOfferingId(offering.getUuid()); + response.setControlOfferingName(offering.getName()); + } else if (WORKER == nodeType) { + response.setWorkerOfferingId(offering.getUuid()); + response.setWorkerOfferingName(offering.getName()); + } else if (ETCD == nodeType) { + response.setEtcdOfferingId(offering.getUuid()); + response.setEtcdOfferingName(offering.getName()); + } + } + @Override public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetesClusterId) { KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); @@ -636,6 +766,20 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne response.setServiceOfferingId(offering.getUuid()); response.setServiceOfferingName(offering.getName()); } + + Long cniConfigId = kubernetesCluster.getCniConfigId(); + if (Objects.nonNull(cniConfigId)) { + UserDataVO cniConfig = userDataDao.findById(cniConfigId); + response.setCniConfigId(cniConfig.getUuid()); + response.setCniConfigName(cniConfig.getName()); + } + setNodeTypeServiceOfferingResponse(response, WORKER, kubernetesCluster.getWorkerNodeServiceOfferingId()); + setNodeTypeServiceOfferingResponse(response, CONTROL, kubernetesCluster.getControlNodeServiceOfferingId()); + setNodeTypeServiceOfferingResponse(response, ETCD, kubernetesCluster.getEtcdNodeServiceOfferingId()); + + if (kubernetesCluster.getEtcdNodeCount() != null) { + response.setEtcdNodes(kubernetesCluster.getEtcdNodeCount()); + } KubernetesSupportedVersionVO version = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); if (version != null) { response.setKubernetesVersionId(version.getUuid()); @@ -671,7 +815,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - List vmResponses = new ArrayList(); + List vmResponses = new ArrayList<>(); List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); ResponseView respView = ResponseView.Restricted; Account caller = CallContext.current().getCallingAccount(); @@ -685,9 +829,31 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (userVM != null) { UserVmResponse vmResponse = ApiDBUtils.newUserVmResponse(respView, responseName, userVM, EnumSet.of(VMDetails.nics), caller); - vmResponses.add(vmResponse); + KubernetesUserVmResponse kubernetesUserVmResponse = new KubernetesUserVmResponse(); + try { + BeanUtils.copyProperties(kubernetesUserVmResponse, vmResponse); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to generate zone metrics response"); + } + kubernetesUserVmResponse.setExternalNode(vmMapVO.isExternalNode()); + kubernetesUserVmResponse.setEtcdNode(vmMapVO.isEtcdNode()); + kubernetesUserVmResponse.setNodeVersion(vmMapVO.getNodeVersion()); + vmResponses.add(kubernetesUserVmResponse); } } + List etcdNodeIds = vmList.stream().filter(KubernetesClusterVmMapVO::isEtcdNode).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List etcdIpIds = new ArrayList<>(); + Map etcdIps = new HashMap<>(); + int etcdNodeSshPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + etcdNodeIds.forEach(id -> { + etcdIpIds.addAll(pfRuleDao.listByVm(id).stream().filter(rule -> rule.getSourcePortStart() == etcdNodeSshPort) + .map(PortForwardingRuleVO::getSourceIpAddressId).collect(Collectors.toList())); + }); + etcdIpIds.forEach(id -> { + IPAddressVO ipAddress = ipAddressDao.findById(id); + etcdIps.put(ipAddress.getUuid(), ipAddress.getAddress().addr()); + }); + response.setEtcdIps(etcdIps); } response.setHasAnnotation(annotationDao.hasAnnotations(kubernetesCluster.getUuid(), AnnotationService.EntityType.KUBERNETES_CLUSTER.name(), accountService.isRootAdmin(caller.getId()))); @@ -697,6 +863,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne response.setMaxSize(kubernetesCluster.getMaxSize()); response.setClusterType(kubernetesCluster.getClusterType()); response.setCreated(kubernetesCluster.getCreated()); + return response; } @@ -785,7 +952,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne BaseCmd.getCommandNameByClass(ScaleKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(StartKubernetesClusterCmd.class), BaseCmd.getCommandNameByClass(StopKubernetesClusterCmd.class), - BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class) + BaseCmd.getCommandNameByClass(UpgradeKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(AddNodesToKubernetesClusterCmd.class), + BaseCmd.getCommandNameByClass(RemoveNodesFromKubernetesClusterCmd.class) ).contains(cmdName); case ExternalManaged: return Arrays.asList( @@ -805,7 +974,6 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final String name = cmd.getName(); final Long zoneId = cmd.getZoneId(); final Long kubernetesVersionId = cmd.getKubernetesVersionId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final Long networkId = cmd.getNetworkId(); final String sshKeyPair = cmd.getSSHKeyPairName(); @@ -816,6 +984,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final String dockerRegistryUrl = cmd.getDockerRegistryUrl(); final Long nodeRootDiskSize = cmd.getNodeRootDiskSize(); final String externalLoadBalancerIpAddress = cmd.getExternalLoadBalancerIpAddress(); + final Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); + final Long defaultServiceOfferingId = cmd.getServiceOfferingId(); if (name == null || name.isEmpty()) { throw new InvalidParameterValueException("Invalid name for the Kubernetes cluster name: " + name); @@ -873,10 +1043,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException(String.format("ISO associated with version ID: %s is not in Ready state for datacenter ID: %s", clusterKubernetesVersion.getUuid(), zone.getUuid())); } - ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("No service offering with ID: " + serviceOfferingId); - } + validateServiceOfferingsForNodeTypes(serviceOfferingNodeTypeMap, defaultServiceOfferingId, cmd.getEtcdNodes(), clusterKubernetesVersion); validateSshKeyPairForKubernetesCreateParameters(sshKeyPair, owner); @@ -884,15 +1051,15 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException(String.format("Invalid value for %s", ApiConstants.NODE_ROOT_DISK_SIZE)); } - if (!validateServiceOffering(serviceOffering, clusterKubernetesVersion)) { - throw new InvalidParameterValueException("Given service offering ID: %s is not suitable for Kubernetes cluster"); - } - validateDockerRegistryParams(dockerRegistryUserName, dockerRegistryPassword, dockerRegistryUrl); Network network = validateAndGetNetworkForKubernetesCreateParameters(networkId); if (StringUtils.isNotEmpty(externalLoadBalancerIpAddress)) { + NsxProviderVO nsxProviderVO = nsxProviderDao.findByZoneId(zone.getId()); + if (Objects.nonNull(nsxProviderVO)) { + throw new InvalidParameterValueException("External load balancer IP address is not supported on NSX-enabled zones"); + } if (!NetUtils.isValidIp4(externalLoadBalancerIpAddress) && !NetUtils.isValidIp6(externalLoadBalancerIpAddress)) { throw new InvalidParameterValueException("Invalid external load balancer IP address"); } @@ -909,8 +1076,40 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } + protected void validateServiceOfferingsForNodeTypes(Map map, + Long defaultServiceOfferingId, + Long etcdNodes, + KubernetesSupportedVersion clusterKubernetesVersion) { + for (String key : CLUSTER_NODES_TYPES_LIST) { + validateServiceOfferingForNode(map, defaultServiceOfferingId, key, etcdNodes, clusterKubernetesVersion); + } + } + + protected void validateServiceOfferingForNode(Map map, + Long defaultServiceOfferingId, + String key, Long etcdNodes, + KubernetesSupportedVersion clusterKubernetesVersion) { + if (ETCD.name().equalsIgnoreCase(key) && (etcdNodes == null || etcdNodes == 0)) { + return; + } + Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId); + ServiceOffering serviceOffering = serviceOfferingId != null ? serviceOfferingDao.findById(serviceOfferingId) : null; + if (serviceOffering == null) { + throw new InvalidParameterValueException("When serviceofferingid is not specified, " + + "service offerings for each node type must be specified in the nodeofferings parameter."); + } + try { + validateServiceOffering(serviceOffering, clusterKubernetesVersion); + } catch (InvalidParameterValueException e) { + String msg = String.format("Given service offering ID: %s for %s nodes is not suitable for the Kubernetes cluster version %s - %s", + serviceOffering, key, clusterKubernetesVersion, e.getMessage()); + logger.error(msg); + throw new InvalidParameterValueException(msg); + } + } + private Network getKubernetesClusterNetworkIfMissing(final String clusterName, final DataCenter zone, final Account owner, final int controlNodesCount, - final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId) throws CloudRuntimeException { + final int nodesCount, final String externalLoadBalancerIpAddress, final Long networkId, final Long asNumber) throws CloudRuntimeException { Network network = null; if (networkId != null) { network = networkDao.findById(networkId); @@ -943,6 +1142,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne network = networkService.createGuestNetwork(networkOffering.getId(), clusterName + "-network", owner.getAccountName() + "-network", owner, physicalNetwork, zone.getId(), ControlledEntity.ACLType.Account); + if (!networkOffering.isForVpc() && NetworkOffering.RoutingMode.Dynamic == networkOffering.getRoutingMode()) { + bgpService.allocateASNumber(zone.getId(), asNumber, network.getId(), null); + } } catch (ConcurrentOperationException | InsufficientCapacityException | ResourceAllocationException e) { logAndThrow(Level.ERROR, String.format("Unable to create network for the Kubernetes cluster: %s", clusterName)); } finally { @@ -1015,12 +1217,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd cmd) { final Long kubernetesClusterId = cmd.getId(); - final Long serviceOfferingId = cmd.getServiceOfferingId(); final Long clusterSize = cmd.getClusterSize(); final List nodeIds = cmd.getNodeIds(); final Boolean isAutoscalingEnabled = cmd.isAutoscalingEnabled(); final Long minSize = cmd.getMinSize(); final Long maxSize = cmd.getMaxSize(); + final Long defaultServiceOfferingId = cmd.getServiceOfferingId(); + final Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); if (kubernetesClusterId == null || kubernetesClusterId < 1L) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID"); @@ -1036,7 +1239,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster : %s", kubernetesCluster.getName())); } - if (serviceOfferingId == null && clusterSize == null && nodeIds == null && isAutoscalingEnabled == null) { + if (defaultServiceOfferingId == null && isAnyNodeOfferingEmpty(serviceOfferingNodeTypeMap) + && clusterSize == null && nodeIds == null && isAutoscalingEnabled == null) { throw new InvalidParameterValueException(String.format("Kubernetes cluster %s cannot be scaled, either service offering or cluster size or nodeids to be removed or autoscaling must be passed", kubernetesCluster.getName())); } @@ -1083,8 +1287,9 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } + Long workerOfferingId = serviceOfferingNodeTypeMap != null ? serviceOfferingNodeTypeMap.getOrDefault(WORKER.name(), null) : null; if (nodeIds != null) { - if (clusterSize != null || serviceOfferingId != null) { + if (clusterSize != null || defaultServiceOfferingId != null || workerOfferingId != null) { throw new InvalidParameterValueException("nodeids can not be passed along with clustersize or service offering"); } List nodes = kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(kubernetesCluster.getId(), nodeIds); @@ -1104,39 +1309,70 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne } } - ServiceOffering serviceOffering = null; - if (serviceOfferingId != null) { - serviceOffering = serviceOfferingDao.findById(serviceOfferingId); - if (serviceOffering == null) { - throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); - } else { - if (serviceOffering.isDynamic()) { - throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster : %s, service offering : %s", kubernetesCluster.getName(), serviceOffering.getName())); - } - if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", - kubernetesCluster.getName(), serviceOffering.getName(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); - } - if (serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d vCPUs", - kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumCpu())); - } - if (serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { - throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d MB RAM", - kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumRamSize())); - } - } - final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || - serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) { - logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s", - serviceOffering.getName(), existingServiceOffering.getName(), kubernetesCluster.getName())); - } - } + validateServiceOfferingsForNodeTypesScale(serviceOfferingNodeTypeMap, defaultServiceOfferingId, kubernetesCluster, clusterVersion); validateKubernetesClusterScaleSize(kubernetesCluster, clusterSize, maxClusterSize, zone); } + protected void validateServiceOfferingsForNodeTypesScale(Map map, Long defaultServiceOfferingId, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) { + for (String key : CLUSTER_NODES_TYPES_LIST) { + Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId); + if (serviceOfferingId != null) { + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + if (serviceOffering == null) { + throw new InvalidParameterValueException("Failed to find service offering ID: " + serviceOfferingId); + } + checkServiceOfferingForNodesScale(serviceOffering, kubernetesCluster, clusterVersion); + Long nodeTypeOfferingId = getExistingServiceOfferingIdForNodeType(key, kubernetesCluster); + if (nodeTypeOfferingId == null) { + nodeTypeOfferingId = kubernetesCluster.getServiceOfferingId(); + } + final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(nodeTypeOfferingId); + if (KubernetesCluster.State.Running.equals(kubernetesCluster.getState()) && (serviceOffering.getRamSize() < existingServiceOffering.getRamSize() || + serviceOffering.getCpu() * serviceOffering.getSpeed() < existingServiceOffering.getCpu() * existingServiceOffering.getSpeed())) { + logAndThrow(Level.WARN, String.format("Kubernetes cluster cannot be scaled down for service offering. Service offering : %s offers lesser resources as compared to service offering : %s of Kubernetes cluster : %s", + serviceOffering.getName(), existingServiceOffering.getName(), kubernetesCluster.getName())); + } + } + } + } + + private Long getExistingServiceOfferingIdForNodeType(String key, KubernetesClusterVO kubernetesCluster) { + if (key.equalsIgnoreCase(WORKER.name())) { + return kubernetesCluster.getWorkerNodeServiceOfferingId(); + } else if (key.equalsIgnoreCase(CONTROL.name())) { + return kubernetesCluster.getControlNodeServiceOfferingId(); + } else if (key.equalsIgnoreCase(ETCD.name())) { + return kubernetesCluster.getEtcdNodeServiceOfferingId(); + } + return kubernetesCluster.getServiceOfferingId(); + } + + protected void checkServiceOfferingForNodesScale(ServiceOffering serviceOffering, KubernetesClusterVO kubernetesCluster, KubernetesSupportedVersion clusterVersion) { + if (serviceOffering.isDynamic()) { + throw new InvalidParameterValueException(String.format("Custom service offerings are not supported for Kubernetes clusters. Kubernetes cluster : %s, service offering : %s", kubernetesCluster.getName(), serviceOffering.getName())); + } + if (serviceOffering.getCpu() < MIN_KUBERNETES_CLUSTER_NODE_CPU || serviceOffering.getRamSize() < MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, Kubernetes cluster template(CoreOS) needs minimum %d vCPUs and %d MB RAM", + kubernetesCluster.getName(), serviceOffering.getName(), MIN_KUBERNETES_CLUSTER_NODE_CPU, MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE)); + } + if (serviceOffering.getCpu() < clusterVersion.getMinimumCpu()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d vCPUs", + kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumCpu())); + } + if (serviceOffering.getRamSize() < clusterVersion.getMinimumRamSize()) { + throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s cannot be scaled with service offering : %s, associated Kubernetes version : %s needs minimum %d MB RAM", + kubernetesCluster.getName(), serviceOffering.getName(), clusterVersion.getName(), clusterVersion.getMinimumRamSize())); + } + } + + protected boolean isAnyNodeOfferingEmpty(Map map) { + if (MapUtils.isEmpty(map)) { + return true; + } + return map.values().stream().anyMatch(Objects::isNull); + } + private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) { // Validate parameters validateEndpointUrl(); @@ -1228,6 +1464,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final long controlNodeCount = cmd.getControlNodes(); final long clusterSize = Objects.requireNonNullElse(cmd.getClusterSize(), 0L); final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + Map nodeTypeOfferingMap = cmd.getServiceOfferingNodeTypeMap(); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); @@ -1280,49 +1517,81 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne final DataCenter zone = dataCenterDao.findById(cmd.getZoneId()); final long controlNodeCount = cmd.getControlNodes(); final long clusterSize = cmd.getClusterSize(); - final long totalNodeCount = controlNodeCount + clusterSize; - final ServiceOffering serviceOffering = serviceOfferingDao.findById(cmd.getServiceOfferingId()); + final long etcdNodes = cmd.getEtcdNodes(); + final Map nodeTypeCount = Map.of(WORKER.name(), clusterSize, + CONTROL.name(), controlNodeCount, ETCD.name(), etcdNodes); final Account owner = accountService.getActiveAccountById(cmd.getEntityOwnerId()); final KubernetesSupportedVersion clusterKubernetesVersion = kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()); + final Hypervisor.HypervisorType hypervisor = cmd.getHypervisorType(); + final Long asNumber = cmd.getAsNumber(); - DeployDestination deployDestination = null; - try { - deployDestination = plan(totalNodeCount, zone, serviceOffering); - } catch (InsufficientCapacityException e) { - logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d nodes cluster in zone : %s with service offering : %s", totalNodeCount, zone.getName(), serviceOffering.getName())); - } - if (deployDestination == null || deployDestination.getCluster() == null) { - logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone : %s", zone.getName())); + Map serviceOfferingNodeTypeMap = cmd.getServiceOfferingNodeTypeMap(); + Long defaultServiceOfferingId = cmd.getServiceOfferingId(); + String accountName = cmd.getAccountName(); + Long domainId = cmd.getDomainId(); + Long accountId = null; + if (Objects.nonNull(accountName) && Objects.nonNull(domainId)) { + Account account = accountDao.findActiveAccount(accountName, domainId); + if (Objects.nonNull(account)) { + accountId = account.getId(); + } } + Hypervisor.HypervisorType hypervisorType = getHypervisorTypeAndValidateNodeDeployments(serviceOfferingNodeTypeMap, defaultServiceOfferingId, nodeTypeCount, zone, domainId, accountId, hypervisor); SecurityGroup securityGroup = null; if (zone.isSecurityGroupEnabled()) { securityGroup = getOrCreateSecurityGroupForAccount(owner); } - final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId()); - final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, deployDestination.getCluster().getHypervisorType()); - final long cores = serviceOffering.getCpu() * (controlNodeCount + clusterSize); - final long memory = serviceOffering.getRamSize() * (controlNodeCount + clusterSize); - + Map templateNodeTypeMap = cmd.getTemplateNodeTypeMap(); + final VMTemplateVO finalTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, DEFAULT); + final VMTemplateVO controlNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, CONTROL); + final VMTemplateVO workerNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, WORKER); + final VMTemplateVO etcdNodeTemplate = getKubernetesServiceTemplate(zone, hypervisorType, templateNodeTypeMap, ETCD); + final Network defaultNetwork = getKubernetesClusterNetworkIfMissing(cmd.getName(), zone, owner, (int)controlNodeCount, (int)clusterSize, cmd.getExternalLoadBalancerIpAddress(), cmd.getNetworkId(), asNumber); final SecurityGroup finalSecurityGroup = securityGroup; final KubernetesClusterVO cluster = Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVO doInTransaction(TransactionStatus status) { + Pair capacityPair = calculateClusterCapacity(serviceOfferingNodeTypeMap, nodeTypeCount, defaultServiceOfferingId); + final long cores = capacityPair.first(); + final long memory = capacityPair.second(); + KubernetesClusterVO newCluster = new KubernetesClusterVO(cmd.getName(), cmd.getDisplayName(), zone.getId(), clusterKubernetesVersion.getId(), - serviceOffering.getId(), finalTemplate.getId(), defaultNetwork.getId(), owner.getDomainId(), - owner.getAccountId(), controlNodeCount, clusterSize, KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, + defaultServiceOfferingId, Objects.nonNull(finalTemplate) ? finalTemplate.getId() : null, + defaultNetwork.getId(), owner.getDomainId(), owner.getAccountId(), controlNodeCount, clusterSize, + KubernetesCluster.State.Created, cmd.getSSHKeyPairName(), cores, memory, cmd.getNodeRootDiskSize(), "", KubernetesCluster.ClusterType.CloudManaged); + newCluster.setCniConfigId(cmd.getCniConfigId()); + String cniConfigDetails = null; + if (MapUtils.isNotEmpty(cmd.getCniConfigDetails())) { + cniConfigDetails = cmd.getCniConfigDetails().toString(); + } + newCluster.setCniConfigDetails(cniConfigDetails); + if (serviceOfferingNodeTypeMap.containsKey(WORKER.name())) { + newCluster.setWorkerNodeServiceOfferingId(serviceOfferingNodeTypeMap.get(WORKER.name())); + } + if (serviceOfferingNodeTypeMap.containsKey(CONTROL.name())) { + newCluster.setControlNodeServiceOfferingId(serviceOfferingNodeTypeMap.get(CONTROL.name())); + } + if (etcdNodes > 0) { + newCluster.setEtcdNodeTemplateId(etcdNodeTemplate.getId()); + newCluster.setEtcdNodeCount(etcdNodes); + if (serviceOfferingNodeTypeMap.containsKey(ETCD.name())) { + newCluster.setEtcdNodeServiceOfferingId(serviceOfferingNodeTypeMap.get(ETCD.name())); + } + } + newCluster.setWorkerNodeTemplateId(workerNodeTemplate.getId()); + newCluster.setControlNodeTemplateId(controlNodeTemplate.getId()); if (zone.isSecurityGroupEnabled()) { newCluster.setSecurityGroupId(finalSecurityGroup.getId()); } kubernetesClusterDao.persist(newCluster); + addKubernetesClusterDetails(newCluster, defaultNetwork, cmd); return newCluster; } }); - addKubernetesClusterDetails(cluster, defaultNetwork, cmd); - if (logger.isInfoEnabled()) { logger.info("Kubernetes cluster {} has been created", cluster); } @@ -1330,6 +1599,59 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return cluster; } + protected Pair calculateClusterCapacity(Map map, Map nodeTypeCount, Long defaultServiceOfferingId) { + long cores = 0L; + long memory = 0L; + for (String key : CLUSTER_NODES_TYPES_LIST) { + if (nodeTypeCount.getOrDefault(key, 0L) == 0) { + continue; + } + Long serviceOfferingId = map.getOrDefault(key, defaultServiceOfferingId); + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + Long nodes = nodeTypeCount.get(key); + cores = cores + (serviceOffering.getCpu() * nodes); + memory = memory + (serviceOffering.getRamSize() * nodes); + } + return new Pair<>(cores, memory); + } + + protected Hypervisor.HypervisorType getHypervisorTypeAndValidateNodeDeployments(Map serviceOfferingNodeTypeMap, + Long defaultServiceOfferingId, + Map nodeTypeCount, + DataCenter zone, Long domainId, Long accountId, + Hypervisor.HypervisorType hypervisorType) { + Hypervisor.HypervisorType deploymentHypervisor = null; + for (String nodeType : CLUSTER_NODES_TYPES_LIST) { + if (!nodeTypeCount.containsKey(nodeType)) { + continue; + } + Long serviceOfferingId = serviceOfferingNodeTypeMap.getOrDefault(nodeType, defaultServiceOfferingId); + ServiceOffering serviceOffering = serviceOfferingDao.findById(serviceOfferingId); + Long nodes = nodeTypeCount.getOrDefault(nodeType, defaultServiceOfferingId); + try { + if (nodeType.equalsIgnoreCase(ETCD.name()) && + (!serviceOfferingNodeTypeMap.containsKey(ETCD.name()) || nodes == 0)) { + continue; + } + DeployDestination deployDestination = plan(nodes, zone, serviceOffering, domainId, accountId, hypervisorType); + if (deployDestination.getCluster() == null) { + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to error while finding suitable deployment plan for cluster in zone : %s", zone.getName())); + } + if (deploymentHypervisor == null) { + deploymentHypervisor = deployDestination.getCluster().getHypervisorType(); + if (hypervisorType != deploymentHypervisor) { + String msg = String.format("The hypervisor type planned for the CKS cluster deployment %s is different " + + "from the selected hypervisor %s", deployDestination.getCluster().getHypervisorType(), hypervisorType); + logger.warn(msg); + } + } + } catch (InsufficientCapacityException e) { + logAndThrow(Level.ERROR, String.format("Creating Kubernetes cluster failed due to insufficient capacity for %d nodes cluster in zone : %s with service offering : %s", nodes, zone.getName(), serviceOffering.getName())); + } + } + return deploymentHypervisor; + } + private SecurityGroup getOrCreateSecurityGroupForAccount(Account owner) { String securityGroupName = String.format("%s-%s", KubernetesClusterActionWorker.CKS_CLUSTER_SECURITY_GROUP_NAME, owner.getUuid()); String securityGroupDesc = String.format("%s and account %s", KubernetesClusterActionWorker.CKS_SECURITY_GROUP_DESCRIPTION, owner.getName()); @@ -1356,7 +1678,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Override @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_CREATE, eventDescription = "creating Kubernetes cluster", async = true) - public void startKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException { + public void startKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { final Long id = cmd.getEntityId(); if (KubernetesCluster.ClusterType.valueOf(cmd.getClusterType()) != KubernetesCluster.ClusterType.CloudManaged) { return; @@ -1365,7 +1687,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (kubernetesCluster == null) { throw new InvalidParameterValueException("Failed to find Kubernetes cluster with given ID"); } - if (!startKubernetesCluster(kubernetesCluster, true)) { + Account account = accountService.getAccount(kubernetesCluster.getAccountId()); + if (!startKubernetesCluster(kubernetesCluster.getId(), kubernetesCluster.getDomainId(), account.getAccountName(), cmd.getAsNumber(), true)) { throw new CloudRuntimeException(String.format("Failed to start created Kubernetes cluster: %s", kubernetesCluster.getName())); } @@ -1374,7 +1697,7 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne @Override @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_START, eventDescription = "starting Kubernetes cluster", async = true) - public void startKubernetesCluster(StartKubernetesClusterCmd cmd) throws CloudRuntimeException { + public void startKubernetesCluster(StartKubernetesClusterCmd cmd) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { final Long id = cmd.getId(); if (id == null || id < 1L) { throw new InvalidParameterValueException("Invalid Kubernetes cluster ID provided"); @@ -1387,7 +1710,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne throw new InvalidParameterValueException(String.format("Start kubernetes cluster is not supported for " + "an externally managed cluster (%s)", kubernetesCluster.getName())); } - if (!startKubernetesCluster(kubernetesCluster, false)) { + Account account = accountService.getAccount(kubernetesCluster.getAccountId()); + if (!startKubernetesCluster(kubernetesCluster.getId(), kubernetesCluster.getDomainId(), account.getAccountName(), null, false)) { throw new CloudRuntimeException(String.format("Failed to start Kubernetes cluster: %s", kubernetesCluster.getName())); } @@ -1399,15 +1723,24 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne * are provisioned from scratch. Second kind of start, happens on Stopped Kubernetes cluster, in which all resources * are provisioned (like volumes, nics, networks etc). It just that VM's are not in running state. So just * start the VM's (which can possibly implicitly start the network also). - * @param kubernetesCluster + * + * @param kubernetesClusterId + * @param domainId + * @param accountName * @param onCreate * @return * @throws CloudRuntimeException */ - public boolean startKubernetesCluster(KubernetesClusterVO kubernetesCluster, boolean onCreate) throws CloudRuntimeException { + @Override + public boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, Long asNumber, boolean onCreate) + throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { if (!KubernetesServiceEnabled.value()) { logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } + final KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Failed to find Kubernetes cluster with given ID"); + } if (kubernetesCluster.getRemoved() != null) { throw new InvalidParameterValueException(String.format("Kubernetes cluster : %s is already deleted", kubernetesCluster.getName())); @@ -1428,6 +1761,13 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne if (zone == null) { logAndThrow(Level.WARN, String.format("Unable to find zone for Kubernetes cluster %s", kubernetesCluster)); } + Long accountId = null; + if (Objects.nonNull(accountName) && Objects.nonNull(domainId)) { + Account account = accountDao.findActiveAccount(accountName, domainId); + if (Objects.nonNull(account)) { + accountId = account.getId(); + } + } KubernetesClusterStartWorker startWorker = new KubernetesClusterStartWorker(kubernetesCluster, this); startWorker = ComponentContext.inject(startWorker); @@ -1435,10 +1775,10 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne // Start for Kubernetes cluster in 'Created' state String[] keys = getServiceUserKeys(kubernetesCluster); startWorker.setKeys(keys); - return startWorker.startKubernetesClusterOnCreate(); + return startWorker.startKubernetesClusterOnCreate(domainId, accountId, asNumber); } else { // Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started - return startWorker.startStoppedKubernetesCluster(); + return startWorker.startStoppedKubernetesCluster(domainId, accountId); } } @@ -1744,28 +2084,47 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); } validateKubernetesClusterScaleParameters(cmd); - KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId()); - final Long clusterSize = cmd.getClusterSize(); - if (clusterSize != null) { - CallContext.current().setEventDetails(String.format("Kubernetes cluster ID: %s scaling from size: %d to %d", - kubernetesCluster.getUuid(), kubernetesCluster.getNodeCount(), clusterSize)); - } + Map nodeToOfferingMap = createNodeTypeToServiceOfferingMap(cmd.getServiceOfferingNodeTypeMap(), cmd.getServiceOfferingId(), kubernetesCluster); + String[] keys = getServiceUserKeys(kubernetesCluster); KubernetesClusterScaleWorker scaleWorker = - new KubernetesClusterScaleWorker(kubernetesClusterDao.findById(cmd.getId()), - serviceOfferingDao.findById(cmd.getServiceOfferingId()), - clusterSize, - cmd.getNodeIds(), - cmd.isAutoscalingEnabled(), - cmd.getMinSize(), - cmd.getMaxSize(), - this); + new KubernetesClusterScaleWorker(kubernetesCluster, + nodeToOfferingMap, + cmd.getClusterSize(), + cmd.getNodeIds(), + cmd.isAutoscalingEnabled(), + cmd.getMinSize(), + cmd.getMaxSize(), + this); scaleWorker.setKeys(keys); scaleWorker = ComponentContext.inject(scaleWorker); return scaleWorker.scaleCluster(); } + /** + * Creates a map for the requested node type service offering + * For the node type DEFAULT: Every node is scaled to the same offering + */ + protected Map createNodeTypeToServiceOfferingMap(Map idsMapping, + Long serviceOfferingId, KubernetesClusterVO kubernetesCluster) { + Map map = new HashMap<>(); + if (MapUtils.isEmpty(idsMapping)) { + ServiceOfferingVO offering = serviceOfferingId != null ? + serviceOfferingDao.findById(serviceOfferingId) : + serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + map.put(DEFAULT.name(), offering); + return map; + } + for (String key : CLUSTER_NODES_TYPES_LIST) { + if (!idsMapping.containsKey(key)) { + continue; + } + map.put(key, serviceOfferingDao.findById(idsMapping.get(key))); + } + return map; + } + @Override @ActionEvent(eventType = KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_UPGRADE, eventDescription = "upgrading Kubernetes cluster", async = true) @@ -1833,6 +2192,77 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne return true; } + @Override + public boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd) { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId()); + long networkId = kubernetesCluster.getNetworkId(); + NetworkVO networkVO = networkDao.findById(networkId); + List validNodeIds = validateNodes(cmd.getNodeIds(), networkId, networkVO.getName(), kubernetesCluster, false); + if (validNodeIds.isEmpty()) { + throw new CloudRuntimeException("No valid nodes found to be added to the Kubernetes cluster"); + } + KubernetesClusterAddWorker addWorker = new KubernetesClusterAddWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + addWorker = ComponentContext.inject(addWorker); + return addWorker.addNodesToCluster(validNodeIds, cmd.isMountCksIsoOnVr(), cmd.isManualUpgrade()); + } + + @Override + public boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception { + if (!KubernetesServiceEnabled.value()) { + logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled"); + } + KubernetesClusterVO kubernetesCluster = validateCluster(cmd.getClusterId()); + List validNodeIds = validateNodes(cmd.getNodeIds(), null, null, kubernetesCluster, true); + if (validNodeIds.isEmpty()) { + throw new CloudRuntimeException("No valid nodes found to be removed from the Kubernetes cluster"); + } + KubernetesClusterRemoveWorker removeWorker = new KubernetesClusterRemoveWorker(kubernetesCluster, KubernetesClusterManagerImpl.this); + removeWorker = ComponentContext.inject(removeWorker); + return removeWorker.removeNodesFromCluster(validNodeIds); + } + + private KubernetesClusterVO validateCluster(long clusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(clusterId); + if (kubernetesCluster == null) { + throw new InvalidParameterValueException("Invalid Kubernetes cluster ID specified"); + } + return kubernetesCluster; + } + + private List validateNodes(List nodeIds, Long networkId, String networkName, KubernetesCluster cluster, boolean removeNodes) { + List validNodeIds = new ArrayList<>(nodeIds); + for (Long id : nodeIds) { + VMInstanceVO node = vmInstanceDao.findById(id); + if (Objects.isNull(node)) { + logger.error(String.format("Failed to find node (physical or virtual machine) with ID: %s", id)); + validNodeIds.remove(id); + } else if (!removeNodes) { + VMTemplateVO template = templateDao.findById(node.getTemplateId()); + if (Objects.isNull(template)) { + logger.error((String.format("Failed to find template with ID: %s", id))); + validNodeIds.remove(id); + } else if (!template.isForCks()) { + logger.error(String.format("Node: %s is deployed with a template that is not marked to be used for CKS", node.getId())); + validNodeIds.remove(id); + } + NicVO nicVO = nicDao.findDefaultNicForVM(id); + if (networkId != nicVO.getNetworkId()) { + logger.error(String.format("Node: %s does not have its default NIC in the kubernetes cluster network: %s", node.getId(), networkName)); + validNodeIds.remove(id); + } + List clusterVmMapVO = kubernetesClusterVmMapDao.listByClusterIdAndVmIdsIn(cluster.getId(), Collections.singletonList(id)); + if (Objects.nonNull(clusterVmMapVO) && !clusterVmMapVO.isEmpty()) { + logger.warn(String.format("Node: %s is already part of the cluster %s", node.getId(), cluster.getName())); + validNodeIds.remove(id); + } + } + } + return validNodeIds; + } + @Override public List removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd) { if (!KubernetesServiceEnabled.value()) { @@ -1967,6 +2397,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne cmdList.add(UpgradeKubernetesClusterCmd.class); cmdList.add(AddVirtualMachinesToKubernetesClusterCmd.class); cmdList.add(RemoveVirtualMachinesFromKubernetesClusterCmd.class); + cmdList.add(AddNodesToKubernetesClusterCmd.class); + cmdList.add(RemoveNodesFromKubernetesClusterCmd.class); return cmdList; } @@ -2261,8 +2693,15 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne KubernetesClusterScaleTimeout, KubernetesClusterUpgradeTimeout, KubernetesClusterUpgradeRetries, + KubernetesClusterAddNodeTimeout, + KubernetesClusterRemoveNodeTimeout, KubernetesClusterExperimentalFeaturesEnabled, - KubernetesMaxClusterSize + KubernetesMaxClusterSize, + KubernetesControlNodeInstallAttemptWait, + KubernetesControlNodeInstallReattempts, + KubernetesWorkerNodeInstallAttemptWait, + KubernetesWorkerNodeInstallReattempts, + KubernetesEtcdNodeStartPort }; } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java index 0c2338465de..a809628a976 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java @@ -16,6 +16,10 @@ // under the License. package com.cloud.kubernetes.cluster; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddNodesToKubernetesClusterCmd; import java.util.List; import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd; @@ -23,6 +27,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernete import org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.GetKubernetesClusterConfigCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ListKubernetesClustersCmd; +import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveNodesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetesClusterCmd; import org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd; @@ -82,6 +87,18 @@ public interface KubernetesClusterService extends PluggableService, Configurable "The number of retries if fail to upgrade kubernetes cluster due to some reasons (e.g. drain node, etcdserver leader changed)", true, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterAddNodeTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.add.node.timeout", + "3600", + "Timeout interval (in seconds) in which an external node (VM / baremetal host) addition to a cluster should be completed", + true, + KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesClusterRemoveNodeTimeout = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.cluster.remove.node.timeout", + "900", + "Timeout interval (in seconds) in which an external node (VM / baremetal host) removal from a cluster should be completed", + true, + KubernetesServiceEnabled.key()); static final ConfigKey KubernetesClusterExperimentalFeaturesEnabled = new ConfigKey("Advanced", Boolean.class, "cloud.kubernetes.cluster.experimental.features.enabled", "false", @@ -95,6 +112,36 @@ public interface KubernetesClusterService extends PluggableService, Configurable true, ConfigKey.Scope.Account, KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesControlNodeInstallAttemptWait = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.control.node.install.attempt.wait.duration", + "15", + "Control Nodes: Time in seconds for the installation process to wait before it re-attempts", + true, + KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesControlNodeInstallReattempts = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.control.node.install.reattempt.count", + "100", + "Control Nodes: Number of times the offline installation of K8S will be re-attempted", + true, + KubernetesServiceEnabled.key()); + final ConfigKey KubernetesWorkerNodeInstallAttemptWait = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.worker.node.install.attempt.wait.duration", + "30", + "Worker Nodes: Time in seconds for the installation process to wait before it re-attempts", + true, + KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesWorkerNodeInstallReattempts = new ConfigKey("Advanced", Long.class, + "cloud.kubernetes.worker.node.install.reattempt.count", + "40", + "Worker Nodes: Number of times the offline installation of K8S will be re-attempted", + true, + KubernetesServiceEnabled.key()); + static final ConfigKey KubernetesEtcdNodeStartPort = new ConfigKey("Advanced", Integer.class, + "cloud.kubernetes.etcd.node.start.port", + "50000", + "Start port for Port forwarding rules for etcd nodes", + true, + KubernetesServiceEnabled.key()); KubernetesCluster findById(final Long id); @@ -102,9 +149,11 @@ public interface KubernetesClusterService extends PluggableService, Configurable KubernetesCluster createManagedKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException; - void startKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException; + void startKubernetesCluster(CreateKubernetesClusterCmd cmd) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException; - void startKubernetesCluster(StartKubernetesClusterCmd cmd) throws CloudRuntimeException; + void startKubernetesCluster(StartKubernetesClusterCmd cmd) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException; + + boolean startKubernetesCluster(long kubernetesClusterId, Long domainId, String accountName, Long asNumber, boolean onCreate) throws CloudRuntimeException, ManagementServerException, ResourceUnavailableException, InsufficientCapacityException; boolean stopKubernetesCluster(StopKubernetesClusterCmd cmd) throws CloudRuntimeException; @@ -124,6 +173,10 @@ public interface KubernetesClusterService extends PluggableService, Configurable boolean addVmsToCluster(AddVirtualMachinesToKubernetesClusterCmd cmd); + boolean addNodesToKubernetesCluster(AddNodesToKubernetesClusterCmd cmd); + + boolean removeNodesFromKubernetesCluster(RemoveNodesFromKubernetesClusterCmd cmd) throws Exception; + List removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd); boolean isDirectAccess(Network network); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java index 01268f42111..79fc15f6898 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVO.java @@ -118,6 +118,33 @@ public class KubernetesClusterVO implements KubernetesCluster { @Column(name = "cluster_type") private ClusterType clusterType; + @Column(name = "control_node_service_offering_id") + private Long controlNodeServiceOfferingId; + + @Column(name = "worker_node_service_offering_id") + private Long workerNodeServiceOfferingId; + + @Column(name = "etcd_node_service_offering_id") + private Long etcdNodeServiceOfferingId; + + @Column(name = "etcd_node_count") + private Long etcdNodeCount; + + @Column(name = "control_node_template_id") + private Long controlNodeTemplateId; + + @Column(name = "worker_node_template_id") + private Long workerNodeTemplateId; + + @Column(name = "etcd_node_template_id") + private Long etcdNodeTemplateId; + + @Column(name = "cni_config_id", nullable = true) + private Long cniConfigId = null; + + @Column(name = "cni_config_details", updatable = true, length = 4096) + private String cniConfigDetails; + @Override public long getId() { return id; @@ -237,7 +264,7 @@ public class KubernetesClusterVO implements KubernetesCluster { @Override public long getTotalNodeCount() { - return this.controlNodeCount + this.nodeCount; + return this.controlNodeCount + this.nodeCount + this.getEtcdNodeCount(); } @Override @@ -414,4 +441,77 @@ public class KubernetesClusterVO implements KubernetesCluster { public Class getEntityType() { return KubernetesCluster.class; } + + public Long getControlNodeServiceOfferingId() { + return controlNodeServiceOfferingId; + } + + public void setControlNodeServiceOfferingId(Long controlNodeServiceOfferingId) { + this.controlNodeServiceOfferingId = controlNodeServiceOfferingId; + } + + public Long getWorkerNodeServiceOfferingId() { + return workerNodeServiceOfferingId; + } + + public void setWorkerNodeServiceOfferingId(Long workerNodeServiceOfferingId) { + this.workerNodeServiceOfferingId = workerNodeServiceOfferingId; + } + + public Long getEtcdNodeServiceOfferingId() { + return etcdNodeServiceOfferingId; + } + + public void setEtcdNodeServiceOfferingId(Long etcdNodeServiceOfferingId) { + this.etcdNodeServiceOfferingId = etcdNodeServiceOfferingId; + } + + public Long getEtcdNodeCount() { + return etcdNodeCount != null ? etcdNodeCount : 0L; + } + + public void setEtcdNodeCount(Long etcdNodeCount) { + this.etcdNodeCount = etcdNodeCount; + } + + public Long getEtcdNodeTemplateId() { + return etcdNodeTemplateId; + } + + public void setEtcdNodeTemplateId(Long etcdNodeTemplateId) { + this.etcdNodeTemplateId = etcdNodeTemplateId; + } + + public Long getWorkerNodeTemplateId() { + return workerNodeTemplateId; + } + + public void setWorkerNodeTemplateId(Long workerNodeTemplateId) { + this.workerNodeTemplateId = workerNodeTemplateId; + } + + public Long getControlNodeTemplateId() { + return controlNodeTemplateId; + } + + public void setControlNodeTemplateId(Long controlNodeTemplateId) { + this.controlNodeTemplateId = controlNodeTemplateId; + } + + public Long getCniConfigId() { + return cniConfigId; + } + + public void setCniConfigId(Long cniConfigId) { + this.cniConfigId = cniConfigId; + } + + public String getCniConfigDetails() { + return cniConfigDetails; + } + + public void setCniConfigDetails(String cniConfigDetails) { + this.cniConfigDetails = cniConfigDetails; + } + } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java index f6126f01be5..6c45c63e16d 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterVmMapVO.java @@ -42,6 +42,18 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { @Column(name = "control_node") boolean controlNode; + @Column(name = "etcd_node") + boolean etcdNode; + + @Column(name = "external_node") + boolean externalNode; + + @Column(name = "manual_upgrade") + boolean manualUpgrade; + + @Column(name = "kubernetes_node_version") + String nodeVersion; + public KubernetesClusterVmMapVO() { } @@ -83,4 +95,36 @@ public class KubernetesClusterVmMapVO implements KubernetesClusterVmMap { public void setControlNode(boolean controlNode) { this.controlNode = controlNode; } + + public boolean isEtcdNode() { + return etcdNode; + } + + public void setEtcdNode(boolean etcdNode) { + this.etcdNode = etcdNode; + } + + public boolean isExternalNode() { + return externalNode; + } + + public void setExternalNode(boolean externalNode) { + this.externalNode = externalNode; + } + + public boolean isManualUpgrade() { + return manualUpgrade; + } + + public void setManualUpgrade(boolean manualUpgrade) { + this.manualUpgrade = manualUpgrade; + } + + public String getNodeVersion() { + return nodeVersion; + } + + public void setNodeVersion(String nodeVersion) { + this.nodeVersion = nodeVersion; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java index d7e6f65ca05..30465c99780 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java @@ -18,10 +18,18 @@ package com.cloud.kubernetes.cluster; import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import javax.inject.Inject; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.framework.config.ConfigKey; @@ -38,6 +46,8 @@ import com.cloud.utils.component.AdapterBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.UserVmManager; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.apache.commons.lang3.ObjectUtils; @@ -52,6 +62,10 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet @Inject private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @Inject + protected ServiceOfferingDao serviceOfferingDao; + @Inject + protected VMTemplateDao vmTemplateDao; + @Inject KubernetesClusterService kubernetesClusterService; protected void setEventTypeEntityDetails(Class eventTypeDefinedClass, Class entityClass) { @@ -110,6 +124,126 @@ public class KubernetesServiceHelperImpl extends AdapterBase implements Kubernet } @Override + public boolean isValidNodeType(String nodeType) { + if (StringUtils.isBlank(nodeType)) { + return false; + } + try { + KubernetesClusterNodeType.valueOf(nodeType.toUpperCase()); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + @Override + public Map getServiceOfferingNodeTypeMap(Map> serviceOfferingNodeTypeMap) { + Map mapping = new HashMap<>(); + if (MapUtils.isNotEmpty(serviceOfferingNodeTypeMap)) { + for (Map entry : serviceOfferingNodeTypeMap.values()) { + processNodeTypeOfferingEntryAndAddToMappingIfValid(entry, mapping); + } + } + return mapping; + } + + protected void checkNodeTypeOfferingEntryCompleteness(String nodeTypeStr, String serviceOfferingUuid) { + if (StringUtils.isAnyEmpty(nodeTypeStr, serviceOfferingUuid)) { + String error = String.format("Incomplete Node Type to Service Offering ID mapping: '%s' -> '%s'", nodeTypeStr, serviceOfferingUuid); + logger.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void checkNodeTypeOfferingEntryValues(String nodeTypeStr, ServiceOffering serviceOffering, String serviceOfferingUuid) { + if (!isValidNodeType(nodeTypeStr)) { + String error = String.format("The provided value '%s' for Node Type is invalid", nodeTypeStr); + logger.error(error); + throw new InvalidParameterValueException(String.format(error)); + } + if (serviceOffering == null) { + String error = String.format("Cannot find a service offering with ID %s", serviceOfferingUuid); + logger.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void addNodeTypeOfferingEntry(String nodeTypeStr, String serviceOfferingUuid, ServiceOffering serviceOffering, Map mapping) { + if (logger.isDebugEnabled()) { + logger.debug("Node Type: '{}' should use Service Offering ID: '{}'", nodeTypeStr, serviceOfferingUuid); + } + KubernetesClusterNodeType nodeType = KubernetesClusterNodeType.valueOf(nodeTypeStr.toUpperCase()); + mapping.put(nodeType.name(), serviceOffering.getId()); + } + + protected void processNodeTypeOfferingEntryAndAddToMappingIfValid(Map entry, Map mapping) { + if (MapUtils.isEmpty(entry)) { + return; + } + String nodeTypeStr = entry.get(VmDetailConstants.CKS_NODE_TYPE); + String serviceOfferingUuid = entry.get(VmDetailConstants.OFFERING); + checkNodeTypeOfferingEntryCompleteness(nodeTypeStr, serviceOfferingUuid); + + ServiceOffering serviceOffering = serviceOfferingDao.findByUuid(serviceOfferingUuid); + checkNodeTypeOfferingEntryValues(nodeTypeStr, serviceOffering, serviceOfferingUuid); + + addNodeTypeOfferingEntry(nodeTypeStr, serviceOfferingUuid, serviceOffering, mapping); + } + + protected void checkNodeTypeTemplateEntryCompleteness(String nodeTypeStr, String templateUuid) { + if (StringUtils.isAnyEmpty(nodeTypeStr, templateUuid)) { + String error = String.format("Incomplete Node Type to template ID mapping: '%s' -> '%s'", nodeTypeStr, templateUuid); + logger.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void checkNodeTypeTemplateEntryValues(String nodeTypeStr, VMTemplateVO template, String templateUuid) { + if (!isValidNodeType(nodeTypeStr)) { + String error = String.format("The provided value '%s' for Node Type is invalid", nodeTypeStr); + logger.error(error); + throw new InvalidParameterValueException(String.format(error)); + } + if (template == null) { + String error = String.format("Cannot find a template with ID %s", templateUuid); + logger.error(error); + throw new InvalidParameterValueException(error); + } + } + + protected void addNodeTypeTemplateEntry(String nodeTypeStr, String templateUuid, VMTemplateVO template, Map mapping) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("Node Type: '%s' should use template ID: '%s'", nodeTypeStr, templateUuid)); + } + KubernetesClusterNodeType nodeType = KubernetesClusterNodeType.valueOf(nodeTypeStr.toUpperCase()); + mapping.put(nodeType.name(), template.getId()); + } + + protected void processNodeTypeTemplateEntryAndAddToMappingIfValid(Map entry, Map mapping) { + if (MapUtils.isEmpty(entry)) { + return; + } + String nodeTypeStr = entry.get(VmDetailConstants.CKS_NODE_TYPE); + String templateUuid = entry.get(VmDetailConstants.TEMPLATE); + checkNodeTypeTemplateEntryCompleteness(nodeTypeStr, templateUuid); + + VMTemplateVO template = vmTemplateDao.findByUuid(templateUuid); + checkNodeTypeTemplateEntryValues(nodeTypeStr, template, templateUuid); + + addNodeTypeTemplateEntry(nodeTypeStr, templateUuid, template, mapping); + } + + @Override + public Map getTemplateNodeTypeMap(Map> templateNodeTypeMap) { + Map mapping = new HashMap<>(); + if (MapUtils.isNotEmpty(templateNodeTypeMap)) { + for (Map entry : templateNodeTypeMap.values()) { + processNodeTypeTemplateEntryAndAddToMappingIfValid(entry, mapping); + } + } + return mapping; + } + public void cleanupForAccount(Account account) { kubernetesClusterService.cleanupForAccount(account); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 29ecde477a4..7ce227dfeb7 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -14,17 +14,24 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - package com.cloud.kubernetes.cluster.actionworkers; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import javax.inject.Inject; @@ -32,13 +39,40 @@ import javax.inject.Inject; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offering.ServiceOffering; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.network.firewall.FirewallService; +import com.cloud.network.rules.FirewallRule; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; +import com.cloud.user.SSHKeyPairVO; +import com.cloud.user.dao.UserDataDao; +import com.cloud.utils.component.ComponentContext; +import com.cloud.utils.db.TransactionCallbackWithException; +import com.cloud.utils.net.Ip; +import com.cloud.vm.Nic; +import com.cloud.vm.NicVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.NicDao; +import com.cloud.vm.UserVmManager; +import org.apache.cloudstack.affinity.AffinityGroupVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; import org.apache.cloudstack.ca.CAManager; import org.apache.cloudstack.config.ApiServiceConfiguration; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.userdata.UserDataManager; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -91,11 +125,14 @@ import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.UserVmService; import com.cloud.vm.UserVmVO; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + public class KubernetesClusterActionWorker { @@ -103,13 +140,18 @@ public class KubernetesClusterActionWorker { public static final int CLUSTER_API_PORT = 6443; public static final int DEFAULT_SSH_PORT = 22; public static final int CLUSTER_NODES_DEFAULT_START_SSH_PORT = 2222; + public static final int ETCD_NODE_CLIENT_REQUEST_PORT = 2379; + public static final int ETCD_NODE_PEER_COMM_PORT = 2380; public static final int CLUSTER_NODES_DEFAULT_SSH_PORT_SG = DEFAULT_SSH_PORT; public static final String CKS_CLUSTER_SECURITY_GROUP_NAME = "CKSSecurityGroup"; public static final String CKS_SECURITY_GROUP_DESCRIPTION = "Security group for CKS nodes"; + public static final String CKS_CONFIG_PATH = "/usr/share/cloudstack-management/cks"; protected Logger logger = LogManager.getLogger(getClass()); + protected final static List CLUSTER_NODES_TYPES_LIST = Arrays.asList(WORKER, CONTROL, ETCD); + protected StateMachine2 _stateMachine = KubernetesCluster.State.getStateMachine(); @Inject @@ -147,6 +189,12 @@ public class KubernetesClusterActionWorker { @Inject protected UserVmService userVmService; @Inject + protected UserDataManager userDataManager; + @Inject + protected UserDataDao userDataDao; + @Inject + protected UserVmManager userVmManager; + @Inject protected VlanDao vlanDao; @Inject protected LaunchPermissionDao launchPermissionDao; @@ -154,6 +202,16 @@ public class KubernetesClusterActionWorker { public ProjectService projectService; @Inject public VpcService vpcService; + @Inject + public PortForwardingRulesDao portForwardingRulesDao; + @Inject + protected RulesService rulesService; + @Inject + protected FirewallService firewallService; + @Inject + private NicDao nicDao; + @Inject + protected AffinityGroupDao affinityGroupDao; protected KubernetesClusterDao kubernetesClusterDao; protected KubernetesClusterVmMapDao kubernetesClusterVmMapDao; @@ -163,6 +221,9 @@ public class KubernetesClusterActionWorker { protected KubernetesCluster kubernetesCluster; protected Account owner; protected VirtualMachineTemplate clusterTemplate; + protected VirtualMachineTemplate controlNodeTemplate; + protected VirtualMachineTemplate workerNodeTemplate; + protected VirtualMachineTemplate etcdTemplate; protected File sshKeyFile; protected String publicIpAddress; protected int sshPort; @@ -171,6 +232,8 @@ public class KubernetesClusterActionWorker { protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret"; protected final String deployProviderScriptFilename = "deploy-provider"; protected final String autoscaleScriptFilename = "autoscale-kube-cluster"; + protected final String validateNodeScript = "validate-cks-node"; + protected final String removeNodeFromClusterScript = "remove-node-from-cluster"; protected final String scriptPath = "/opt/bin/"; protected File deploySecretsScriptFile; protected File deployProviderScriptFile; @@ -194,7 +257,10 @@ public class KubernetesClusterActionWorker { DataCenterVO dataCenterVO = dataCenterDao.findById(zoneId); VMTemplateVO template = templateDao.findById(templateId); Hypervisor.HypervisorType type = template.getHypervisorType(); - this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type); + this.clusterTemplate = manager.getKubernetesServiceTemplate(dataCenterVO, type, null, KubernetesClusterNodeType.DEFAULT); + this.controlNodeTemplate = templateDao.findById(this.kubernetesCluster.getControlNodeTemplateId()); + this.workerNodeTemplate = templateDao.findById(this.kubernetesCluster.getWorkerNodeTemplateId()); + this.etcdTemplate = templateDao.findById(this.kubernetesCluster.getEtcdNodeTemplateId()); this.sshKeyFile = getManagementServerSshPublicKeyFile(); } @@ -202,6 +268,11 @@ public class KubernetesClusterActionWorker { return IOUtils.toString(Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)), com.cloud.utils.StringUtils.getPreferredCharset()); } + protected String readK8sConfigFile(String resource) throws IOException { + Path path = Paths.get(String.format("%s%s", CKS_CONFIG_PATH, resource)); + return Files.readString(path); + } + protected String getControlNodeLoginUser() { List vmMapVOList = getKubernetesClusterVMMaps(); if (!vmMapVOList.isEmpty()) { @@ -254,7 +325,7 @@ public class KubernetesClusterActionWorker { } protected void logTransitStateDetachIsoAndThrow(final Level logLevel, final String message, final KubernetesCluster kubernetesCluster, - final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { + final List clusterVMs, final KubernetesCluster.Event event, final Exception e) throws CloudRuntimeException { logMessage(logLevel, message, e); stateTransitTo(kubernetesCluster.getId(), event); detachIsoKubernetesVMs(clusterVMs); @@ -265,7 +336,7 @@ public class KubernetesClusterActionWorker { } protected void deleteTemplateLaunchPermission() { - if (clusterTemplate != null && owner != null) { + if (isDefaultTemplateUsed() && owner != null) { logger.info("Revoking launch permission for systemVM template"); launchPermissionDao.removePermissions(clusterTemplate.getId(), Collections.singletonList(owner.getId())); } @@ -304,11 +375,20 @@ public class KubernetesClusterActionWorker { return new File(keyFile); } - protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, boolean isControlNode) { - return Transaction.execute(new TransactionCallback<>() { + protected KubernetesClusterVmMapVO addKubernetesClusterVm(final long kubernetesClusterId, final long vmId, + boolean isControlNode, boolean isExternalNode, + boolean isEtcdNode, boolean markForManualUpgrade) { + KubernetesSupportedVersion kubernetesVersion = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()); + return Transaction.execute(new TransactionCallback() { @Override public KubernetesClusterVmMapVO doInTransaction(TransactionStatus status) { KubernetesClusterVmMapVO newClusterVmMap = new KubernetesClusterVmMapVO(kubernetesClusterId, vmId, isControlNode); + newClusterVmMap.setExternalNode(isExternalNode); + newClusterVmMap.setManualUpgrade(markForManualUpgrade); + newClusterVmMap.setEtcdNode(isEtcdNode); + if (!isEtcdNode) { + newClusterVmMap.setNodeVersion(kubernetesVersion.getSemanticVersion()); + } kubernetesClusterVmMapDao.persist(newClusterVmMap); return newClusterVmMap; } @@ -319,6 +399,7 @@ public class KubernetesClusterActionWorker { if (controlVm != null) { return controlVm; } + Long etcdNodeCount = kubernetesCluster.getEtcdNodeCount(); List clusterVMs = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); if (CollectionUtils.isEmpty(clusterVMs)) { logger.warn(String.format("Unable to retrieve VMs for Kubernetes cluster : %s", kubernetesCluster.getName())); @@ -329,7 +410,8 @@ public class KubernetesClusterActionWorker { vmIds.add(vmMap.getVmId()); } Collections.sort(vmIds); - return userVmDao.findById(vmIds.get(0)); + int controlNodeIndex = Objects.nonNull(etcdNodeCount) ? etcdNodeCount.intValue() : 0; + return userVmDao.findById(vmIds.get(controlNodeIndex)); } protected String getControlVmPrivateIp() { @@ -368,7 +450,23 @@ public class KubernetesClusterActionWorker { return address; } - protected IpAddress acquireVpcTierKubernetesPublicIp(Network network) throws + protected IpAddress getPublicIp(Network network) throws ManagementServerException { + if (network.getVpcId() != null) { + IpAddress publicIp = getVpcTierKubernetesPublicIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No public IP addresses found for VPC tier : %s, Kubernetes cluster : %s", network.getName(), kubernetesCluster.getName())); + } + return publicIp; + } + IpAddress publicIp = getNetworkSourceNatIp(network); + if (publicIp == null) { + throw new ManagementServerException(String.format("No source NAT IP addresses found for network : %s, Kubernetes cluster : %s", + network.getName(), kubernetesCluster.getName())); + } + return publicIp; + } + + protected IpAddress acquireVpcTierKubernetesPublicIp(Network network, boolean forEtcd) throws InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException { IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null); if (ip == null) { @@ -376,7 +474,19 @@ public class KubernetesClusterActionWorker { } ip = vpcService.associateIPToVpc(ip.getId(), network.getVpcId()); ip = ipAddressManager.associateIPToGuestNetwork(ip.getId(), network.getId(), false); - kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false); + if (!forEtcd) { + kubernetesClusterDetailsDao.addDetail(kubernetesCluster.getId(), ApiConstants.PUBLIC_IP_ID, ip.getUuid(), false); + } + return ip; + } + + protected IpAddress acquirePublicIpForIsolatedNetwork(Network network) throws + InsufficientAddressCapacityException, ResourceAllocationException, ResourceUnavailableException { + IpAddress ip = networkService.allocateIP(owner, kubernetesCluster.getZoneId(), network.getId(), null, null); + if (ip == null) { + return null; + } + ip = networkService.associateIPToNetwork(ip.getId(), network.getId()); return ip; } @@ -408,7 +518,7 @@ public class KubernetesClusterActionWorker { return new Pair<>(address.getAddress().addr(), port); } if (acquireNewPublicIpForVpcTierIfNeeded) { - address = acquireVpcTierKubernetesPublicIp(network); + address = acquireVpcTierKubernetesPublicIp(network, false); if (address != null) { return new Pair<>(address.getAddress().addr(), port); } @@ -501,7 +611,7 @@ public class KubernetesClusterActionWorker { CallContext vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine); vmContext.putContextParameter(VirtualMachine.class, vm.getUuid()); try { - result = templateService.detachIso(vm.getId(), true); + result = templateService.detachIso(vm.getId(), null, true); } catch (CloudRuntimeException ex) { logger.warn("Failed to detach binaries ISO from VM: {} in the Kubernetes cluster: {} ", vm, kubernetesCluster, ex); } finally { @@ -569,14 +679,14 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'", - scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); + scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]); Account account = accountDao.findById(kubernetesCluster.getAccountId()); if (account != null && account.getType() == Account.Type.PROJECT) { String projectId = projectService.findByProjectAccountId(account.getId()).getUuid(); command = String.format("%s -p '%s'", command, projectId); } Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); return result.first(); } catch (Exception e) { String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName()); @@ -595,7 +705,7 @@ public class KubernetesClusterActionWorker { writer.close(); } catch (IOException e) { logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to fetch script %s", - kubernetesCluster.getName(), filename), e); + kubernetesCluster.getName(), filename), e); } return file; } @@ -612,13 +722,17 @@ public class KubernetesClusterActionWorker { copyScriptFile(nodeAddress, sshPort, autoscaleScriptFile, autoscaleScriptFilename); } - protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String desitnation) { + protected void copyScriptFile(String nodeAddress, final int sshPort, File file, String destination) { try { + if (Objects.isNull(sshKeyFile)) { + sshKeyFile = getManagementServerSshPublicKeyFile(); + } SshHelper.scpTo(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - "~/", file.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo mv ~/%s %s/%s", file.getName(), scriptPath, desitnation); - SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, - cmdStr, 10000, 10000, 10 * 60 * 1000); + "~/", file.getAbsolutePath(), "0755", 20000, 30 * 60 * 1000); + // Ensure destination dir scriptPath exists and copy file to destination + String cmdStr = String.format("sudo mkdir -p %s ; sudo mv ~/%s %s/%s", scriptPath, file.getName(), scriptPath, destination); + SshHelper.sshExecute(nodeAddress, sshPort, getControlNodeLoginUser(), sshKeyFile, null, + cmdStr, 10000, 10000, 10 * 60 * 1000); } catch (Exception e) { throw new CloudRuntimeException(e); } @@ -635,20 +749,30 @@ public class KubernetesClusterActionWorker { String command = String.format("sudo /opt/bin/kubectl annotate node %s cluster-autoscaler.kubernetes.io/scale-down-disabled=true ; ", name); commands.append(command); } - try { - File pkFile = getManagementServerSshPublicKeyFile(); - Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); - publicIpAddress = publicIpSshPort.first(); - sshPort = publicIpSshPort.second(); + int retryCounter = 0; + while (retryCounter < 3) { + retryCounter++; + try { + File pkFile = getManagementServerSshPublicKeyFile(); + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + publicIpAddress = publicIpSshPort.first(); + sshPort = publicIpSshPort.second(); - Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, commands.toString(), 10000, 10000, 60000); - return result.first(); - } catch (Exception e) { - String msg = String.format("Failed to taint control nodes on : %s : %s", kubernetesCluster.getName(), e.getMessage()); - logMessage(Level.ERROR, msg, e); - return false; + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), + pkFile, null, commands.toString(), 10000, 10000, 60000); + return result.first(); + } catch (Exception e) { + String msg = String.format("Failed to taint control nodes on : %s : %s", kubernetesCluster.getName(), e.getMessage()); + logMessage(Level.ERROR, msg, e); + } + try { + Thread.sleep(5 * 1000L); + } catch (InterruptedException ie) { + logger.error(String.format("Error while attempting to taint nodes on Kubernetes cluster: %s", kubernetesCluster.getName()), ie); + } + retryCounter++; } + return false; } protected boolean deployProvider() { @@ -656,7 +780,7 @@ public class KubernetesClusterActionWorker { // Since the provider creates IP addresses, don't deploy it unless the underlying network supports it if (manager.isDirectAccess(network)) { logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network", - kubernetesCluster.getName()), null); + kubernetesCluster.getName()), null); return true; } File pkFile = getManagementServerSshPublicKeyFile(); @@ -667,7 +791,7 @@ public class KubernetesClusterActionWorker { try { String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename); Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); // Maybe the file isn't present. Try and copy it if (!result.first()) { @@ -677,12 +801,12 @@ public class KubernetesClusterActionWorker { if (!createCloudStackSecret(keys)) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s", - kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } // If at first you don't succeed ... result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), - pkFile, null, command, 10000, 10000, 60000); + pkFile, null, command, 10000, 10000, 60000); if (!result.first()) { throw new CloudRuntimeException(result.second()); } @@ -698,4 +822,247 @@ public class KubernetesClusterActionWorker { public void setKeys(String[] keys) { this.keys = keys; } + + protected ServiceOffering getServiceOfferingForNodeTypeOnCluster(KubernetesClusterNodeType nodeType, + KubernetesCluster cluster) { + Long offeringId = null; + Long defaultOfferingId = cluster.getServiceOfferingId(); + Long controlOfferingId = cluster.getControlNodeServiceOfferingId(); + Long workerOfferingId = cluster.getWorkerNodeServiceOfferingId(); + Long etcdOfferingId = cluster.getEtcdNodeServiceOfferingId(); + if (KubernetesClusterNodeType.CONTROL == nodeType) { + offeringId = controlOfferingId != null ? controlOfferingId : defaultOfferingId; + } else if (KubernetesClusterNodeType.WORKER == nodeType) { + offeringId = workerOfferingId != null ? workerOfferingId : defaultOfferingId; + } else if (KubernetesClusterNodeType.ETCD == nodeType && cluster.getEtcdNodeCount() != null && cluster.getEtcdNodeCount() > 0) { + offeringId = etcdOfferingId != null ? etcdOfferingId : defaultOfferingId; + } + + if (offeringId == null) { + String msg = String.format("Cannot find a service offering for the %s nodes on the Kubernetes cluster %s", nodeType.name(), cluster.getName()); + logger.error(msg); + throw new CloudRuntimeException(msg); + } + return serviceOfferingDao.findById(offeringId); + } + + protected boolean isDefaultTemplateUsed() { + if (Arrays.asList(kubernetesCluster.getControlNodeTemplateId(), kubernetesCluster.getWorkerNodeTemplateId(), kubernetesCluster.getEtcdNodeTemplateId()).contains(kubernetesCluster.getTemplateId())) { + return true; + } + return false; + } + + protected void provisionPublicIpPortForwardingRule(IpAddress publicIp, Network network, Account account, + final long vmId, final int sourcePort, final int destPort) throws NetworkRuleConflictException, ResourceUnavailableException { + final long publicIpId = publicIp.getId(); + final long networkId = network.getId(); + final long accountId = account.getId(); + final long domainId = account.getDomainId(); + Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); + final Ip vmIp = new Ip(vmNic.getIPv4Address()); + PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException) status -> { + PortForwardingRuleVO newRule = + new PortForwardingRuleVO(null, publicIpId, + sourcePort, sourcePort, + vmIp, + destPort, destPort, + "tcp", networkId, accountId, domainId, vmId); + newRule.setDisplay(true); + newRule.setState(FirewallRule.State.Add); + newRule = portForwardingRulesDao.persist(newRule); + return newRule; + }); + rulesService.applyPortForwardingRules(publicIp.getId(), account); + if (logger.isInfoEnabled()) { + logger.info(String.format("Provisioned SSH port forwarding rule: %s from port %d to %d on %s to the VM IP : %s in Kubernetes cluster : %s", pfRule.getUuid(), sourcePort, destPort, publicIp.getAddress().addr(), vmIp.toString(), kubernetesCluster.getName())); + } + } + + public String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso, final boolean mountCksIsoOnVR) throws IOException { + String k8sNodeConfig = readK8sConfigFile("/conf/k8s-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String joinIpKey = "{{ k8s_control_node.join_ip }}"; + final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; + final String routerIpKey = "{{ k8s.vr.iso.mounted.ip }}"; + final String installWaitTime = "{{ k8s.install.wait.time }}"; + final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + + final Long waitTime = KubernetesClusterService.KubernetesWorkerNodeInstallAttemptWait.value(); + final Long reattempts = KubernetesClusterService.KubernetesWorkerNodeInstallReattempts.value(); + String routerIp = ""; + if (mountCksIsoOnVR) { + NicVO routerNicOnNetwork = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster); + if (Objects.nonNull(routerNicOnNetwork)) { + routerIp = routerNicOnNetwork.getIPv4Address(); + } + } + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (StringUtils.isNotEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); + k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); + k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sNodeConfig = k8sNodeConfig.replace(routerIpKey, routerIp); + k8sNodeConfig = k8sNodeConfig.replace(installWaitTime, String.valueOf(waitTime)); + k8sNodeConfig = k8sNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); + + k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); + + return k8sNodeConfig; + } + + protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { + /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to + * use docker private registry */ + String registryUsername = null; + String registryPassword = null; + String registryUrl = null; + + List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); + for (KubernetesClusterDetailsVO detail : details) { + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { + registryUsername = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { + registryPassword = detail.getValue(); + } + if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { + registryUrl = detail.getValue(); + } + } + + if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) { + // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details + String runCmd = "- bash -x /opt/bin/setup-containerd"; + + String registryEp = registryUrl.split("://")[1]; + k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); + final String registryUrlKey = "{{registry.url}}"; + final String registryUrlEpKey = "{{registry.url.endpoint}}"; + final String registryAuthKey = "{{registry.token}}"; + final String registryUname = "{{registry.username}}"; + final String registryPsswd = "{{registry.password}}"; + + final String usernamePasswordKey = registryUsername + ":" + registryPassword; + String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); + k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); + k8sConfig = k8sConfig.replace(registryUname, registryUsername); + k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); + k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); + } + return k8sConfig; + } + + public Map addFirewallRulesForNodes(IpAddress publicIp, int size) throws ManagementServerException { + Map vmIdPortMap = new HashMap<>(); + CallContext.register(CallContext.current(), null); + try { + List clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); + int endPort = (CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVmList.size() - externalNodes.size() - kubernetesCluster.getEtcdNodeCount().intValue() - 1); + provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); + if (logger.isInfoEnabled()) { + logger.info(String.format("Provisioned firewall rule to open up port %d to %d on %s for Kubernetes cluster : %s", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster.getName())); + } + if (!externalNodes.isEmpty()) { + AtomicInteger additionalNodes = new AtomicInteger(1); + externalNodes.forEach(externalNode -> { + int port = endPort + additionalNodes.get(); + try { + provisionFirewallRules(publicIp, owner, port, port); + vmIdPortMap.put(externalNode.getVmId(), port); + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new CloudRuntimeException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + additionalNodes.addAndGet(1); + }); + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } finally { + CallContext.unregister(); + } + return vmIdPortMap; + } + + protected void provisionFirewallRules(final IpAddress publicIp, final Account account, int startPort, int endPort) throws NoSuchFieldException, + IllegalAccessException, ResourceUnavailableException, NetworkRuleConflictException { + List sourceCidrList = new ArrayList(); + sourceCidrList.add("0.0.0.0/0"); + + CreateFirewallRuleCmd rule = new CreateFirewallRuleCmd(); + rule = ComponentContext.inject(rule); + + Field addressField = rule.getClass().getDeclaredField("ipAddressId"); + addressField.setAccessible(true); + addressField.set(rule, publicIp.getId()); + + Field protocolField = rule.getClass().getDeclaredField("protocol"); + protocolField.setAccessible(true); + protocolField.set(rule, "TCP"); + + Field startPortField = rule.getClass().getDeclaredField("publicStartPort"); + startPortField.setAccessible(true); + startPortField.set(rule, startPort); + + Field endPortField = rule.getClass().getDeclaredField("publicEndPort"); + endPortField.setAccessible(true); + endPortField.set(rule, endPort); + + Field cidrField = rule.getClass().getDeclaredField("cidrlist"); + cidrField.setAccessible(true); + cidrField.set(rule, sourceCidrList); + + firewallService.createIngressFirewallRule(rule); + firewallService.applyIngressFwRules(publicIp.getId(), account); + } + + protected NicVO getVirtualRouterNicOnKubernetesClusterNetwork(KubernetesCluster kubernetesCluster) { + long networkId = kubernetesCluster.getNetworkId(); + NetworkVO kubernetesClusterNetwork = networkDao.findById(networkId); + if (kubernetesClusterNetwork == null) { + logAndThrow(Level.ERROR, String.format("Cannot find network %s set on Kubernetes Cluster %s", networkId, kubernetesCluster.getName())); + } + NicVO routerNicOnNetwork = nicDao.findByNetworkIdAndType(networkId, VirtualMachine.Type.DomainRouter); + if (routerNicOnNetwork == null) { + logAndThrow(Level.ERROR, String.format("Cannot find a Virtual Router on Kubernetes Cluster %s network %s", kubernetesCluster.getName(), kubernetesClusterNetwork.getName())); + } + return routerNicOnNetwork; + } + + protected Map getVmPortMap() { + List clusterVmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + List externalNodes = clusterVmList.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); + Map vmIdPortMap = new HashMap<>(); + int defaultNodesCount = clusterVmList.size() - externalNodes.size(); + AtomicInteger i = new AtomicInteger(0); + externalNodes.forEach(node -> { + vmIdPortMap.put(node.getVmId(), CLUSTER_NODES_DEFAULT_START_SSH_PORT + defaultNodesCount + i.get()); + i.addAndGet(1); + }); + return vmIdPortMap; + } + + public Long getExplicitAffinityGroup(Long domainId, Long accountId) { + AffinityGroupVO groupVO = null; + if (Objects.nonNull(accountId)) { + groupVO = affinityGroupDao.findByAccountAndType(accountId, "ExplicitDedication"); + } + if (Objects.isNull(groupVO)) { + groupVO = affinityGroupDao.findDomainLevelGroupByType(domainId, "ExplicitDedication"); + } + if (Objects.nonNull(groupVO)) { + return groupVO.getId(); + } + return null; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java new file mode 100644 index 00000000000..8b694adf1cc --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterAddWorker.java @@ -0,0 +1,326 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventVO; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.Ternary; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmVO; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.user.vm.RebootVMCmd; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.codec.binary.Base64; +import org.apache.logging.log4j.Level; + +import javax.inject.Inject; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class KubernetesClusterAddWorker extends KubernetesClusterActionWorker { + + @Inject + private FirewallRulesDao firewallRulesDao; + private long addNodeTimeoutTime; + + List finalNodeIds = new ArrayList<>(); + + public KubernetesClusterAddWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + public boolean addNodesToCluster(List nodeIds, boolean mountCksIsoOnVr, boolean manualUpgrade) throws CloudRuntimeException { + try { + init(); + addNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterAddNodeTimeout.value() * 1000; + Long networkId = kubernetesCluster.getNetworkId(); + Network network = networkDao.findById(networkId); + if (Objects.isNull(network)) { + throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); + } + templateDao.findById(kubernetesCluster.getTemplateId()); + IpAddress publicIp = null; + try { + publicIp = getPublicIp(network); + } catch (ManagementServerException e) { + throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); + } + attachCksIsoForNodesAdditionToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.AddNodeRequested); + String controlNodeGuestIp = getControlVmPrivateIp(); + Ternary nodesAddedAndMemory = importNodeToCluster(nodeIds, network, publicIp, controlNodeGuestIp, mountCksIsoOnVr); + int nodesAdded = nodesAddedAndMemory.first(); + updateKubernetesCluster(kubernetesCluster.getId(), nodesAddedAndMemory, manualUpgrade); + if (nodeIds.size() != nodesAdded) { + String msg = String.format("Not every node was added to the CKS cluster %s, nodes added: %s out of %s", kubernetesCluster.getUuid(), nodesAdded, nodeIds.size()); + logger.info(msg); + detachCksIsoFromNodesAddedToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_ERROR, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD, + msg, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + return false; + } + Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); + KubernetesClusterUtil.validateKubernetesClusterReadyNodesCount(kubernetesCluster, publicIpSshPort.first(), publicIpSshPort.second(), + getControlNodeLoginUser(), sshKeyFile, addNodeTimeoutTime, 15000); + detachCksIsoFromNodesAddedToCluster(nodeIds, kubernetesCluster.getId(), mountCksIsoOnVr); + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + String description = String.format("Successfully added %s nodes to Kubernetes Cluster %s", nodesAdded, kubernetesCluster.getUuid()); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD, + description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + return true; + } catch (Exception e) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + throw new CloudRuntimeException(e); + } + } + + private void detachCksIsoFromNodesAddedToCluster(List nodeIds, long kubernetesClusterId, boolean mountCksIsoOnVr) { + if (mountCksIsoOnVr) { + detachIsoOnVirtualRouter(kubernetesClusterId); + } else { + logger.info("Detaching CKS ISO from the nodes"); + List vms = nodeIds.stream().map(nodeId -> userVmDao.findById(nodeId)).collect(Collectors.toList()); + detachIsoKubernetesVMs(vms); + } + } + + public void detachIsoOnVirtualRouter(Long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + Long virtualRouterId = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster).getInstanceId(); + long isoId = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getIsoId(); + + try { + networkService.handleCksIsoOnNetworkVirtualRouter(virtualRouterId, false); + } catch (ResourceUnavailableException e) { + String err = String.format("Error trying to handle ISO %s on virtual router %s", isoId, virtualRouterId); + logger.error(err); + throw new CloudRuntimeException(err); + } + + try { + templateService.detachIso(virtualRouterId, isoId, true, true); + } catch (CloudRuntimeException e) { + String err = String.format("Error trying to detach ISO %s from virtual router %s", isoId, virtualRouterId); + logger.error(err, e); + } + } + + public void attachCksIsoForNodesAdditionToCluster(List nodeIds, Long kubernetesClusterId, boolean mountCksIsoOnVr) { + if (mountCksIsoOnVr) { + attachAndServeIsoOnVirtualRouter(kubernetesClusterId); + } else { + logger.info("Attaching CKS ISO to the nodes"); + List vms = nodeIds.stream().map(nodeId -> userVmDao.findById(nodeId)).collect(Collectors.toList()); + attachIsoKubernetesVMs(vms); + } + } + + public void attachAndServeIsoOnVirtualRouter(Long kubernetesClusterId) { + KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(kubernetesClusterId); + Long virtualRouterId = getVirtualRouterNicOnKubernetesClusterNetwork(kubernetesCluster).getInstanceId(); + long isoId = kubernetesSupportedVersionDao.findById(kubernetesCluster.getKubernetesVersionId()).getIsoId(); + + try { + templateService.attachIso(isoId, virtualRouterId, true, true); + } catch (CloudRuntimeException e) { + String err = String.format("Error trying to attach ISO %s to virtual router %s", isoId, virtualRouterId); + logger.error(err); + throw new CloudRuntimeException(err); + } + + try { + networkService.handleCksIsoOnNetworkVirtualRouter(virtualRouterId, true); + } catch (ResourceUnavailableException e) { + String err = String.format("Error trying to handle ISO %s on virtual router %s", isoId, virtualRouterId); + logger.error(err); + throw new CloudRuntimeException(err); + } + } + + private Ternary importNodeToCluster(List nodeIds, Network network, IpAddress publicIp, + String controlNodeGuestIp, boolean mountCksIsoOnVr) { + int nodeIndex = 0; + Long additionalMemory = 0L; + Long additionalCores = 0L; + for (Long nodeId : nodeIds) { + UserVmVO vm = userVmDao.findById(nodeId); + String k8sControlNodeConfig = null; + try { + k8sControlNodeConfig = getKubernetesNodeConfig(controlNodeGuestIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), mountCksIsoOnVr); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); + } + if (Objects.isNull(k8sControlNodeConfig)) { + logAndThrow(Level.ERROR, "Error generating worker node configuration"); + } + String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + + Pair result = validateAndSetupNode(network, publicIp, owner, nodeId, nodeIndex, base64UserData); + if (Boolean.TRUE.equals(result.first())) { + ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + additionalMemory += offeringVO.getRamSize(); + additionalCores += offeringVO.getCpu(); + String msg = String.format("VM %s added as a node on the Kubernetes Cluster %s", vm.getUuid(), kubernetesCluster.getUuid()); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD, + msg, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), 0); + } + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Failed to add node %s [%s] to Kubernetes cluster : %s", vm.getName(), vm.getUuid(), kubernetesCluster.getName())); + } + if (System.currentTimeMillis() > addNodeTimeoutTime) { + logger.error(String.format("Failed to add node %s to Kubernetes cluster : %s", nodeId, kubernetesCluster.getName())); + } + nodeIndex = result.second(); + } + return new Ternary<>(nodeIndex, additionalMemory, additionalCores); + } + + private Pair validateAndSetupNode(Network network, IpAddress publicIp, Account account, + Long nodeId, int nodeIndex, String base64UserData) { + int startSshPortNumber = KubernetesClusterActionWorker.CLUSTER_NODES_DEFAULT_START_SSH_PORT + (int) kubernetesCluster.getTotalNodeCount() - kubernetesCluster.getEtcdNodeCount().intValue(); + int sshStartPort = startSshPortNumber + nodeIndex; + try { + if (Objects.isNull(network.getVpcId())) { + provisionFirewallRules(publicIp, owner, sshStartPort, sshStartPort); + } + provisionPublicIpPortForwardingRule(publicIp, network, account, nodeId, sshStartPort, DEFAULT_SSH_PORT); + boolean isCompatible = validateNodeCompatibility(publicIp, nodeId, sshStartPort); + if (!isCompatible) { + revertNetworkRules(network, nodeId, sshStartPort); + return new Pair<>(false, nodeIndex); + } + + userVmManager.updateVirtualMachine(nodeId, null, null, null, null, + null, null, base64UserData, null, null, null, + BaseCmd.HTTPMethod.POST, null, null, null, null, null); + + RebootVMCmd rebootVMCmd = new RebootVMCmd(); + Field idField = rebootVMCmd.getClass().getDeclaredField("id"); + idField.setAccessible(true); + idField.set(rebootVMCmd, nodeId); + userVmService.rebootVirtualMachine(rebootVMCmd); + finalNodeIds.add(nodeId); + } catch (ResourceUnavailableException | NetworkRuleConflictException | NoSuchFieldException | + InsufficientCapacityException | IllegalAccessException e) { + logger.error(String.format("Failed to activate API port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName())); + // remove added Firewall and PF rules + revertNetworkRules(network, nodeId, sshStartPort); + return new Pair<>( false, nodeIndex); + } catch (Exception e) { + String errMsg = String.format("Unexpected exception while trying to add the external node %s to the Kubernetes cluster %s: %s", + nodeId, kubernetesCluster.getName(), e.getMessage()); + logger.error(errMsg, e); + revertNetworkRules(network, nodeId, sshStartPort); + throw new CloudRuntimeException(e); + } + return new Pair<>(true, ++nodeIndex); + } + + private void updateKubernetesCluster(long clusterId, Ternary additionalNodesDetails, boolean manualUpgrade) { + int additionalNodeCount = additionalNodesDetails.first(); + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId); + kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() + additionalNodeCount); + kubernetesClusterVO.setMemory(kubernetesClusterVO.getMemory() + additionalNodesDetails.second()); + kubernetesClusterVO.setCores(kubernetesClusterVO.getCores() + additionalNodesDetails.third()); + kubernetesClusterDao.update(clusterId, kubernetesClusterVO); + kubernetesCluster = kubernetesClusterVO; + + finalNodeIds.forEach(id -> addKubernetesClusterVm(clusterId, id, false, true, false, manualUpgrade)); + } + + + private boolean validateNodeCompatibility(IpAddress publicIp, long nodeId, int nodeSshPort) throws CloudRuntimeException { + File pkFile = getManagementServerSshPublicKeyFile(); + try { + File validateNodeScriptFile = retrieveScriptFile(validateNodeScript); + Thread.sleep(15*1000); + copyScriptFile(publicIp.getAddress().addr(), nodeSshPort, validateNodeScriptFile, validateNodeScript); + String command = String.format("%s%s", scriptPath, validateNodeScript); + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Node with ID: %s cannot be added as a worker node as it does not have " + + "the following dependencies: %s ", nodeId, result.second())); + return false; + } + } catch (Exception e) { + logger.error(String.format("Failed to validate node with ID: %s", nodeId), e); + return false; + } + UserVmVO userVm = userVmDao.findById(nodeId); + cleanupCloudInitSemFolder(userVm, publicIp, pkFile, nodeSshPort); + return true; + } + + private void cleanupCloudInitSemFolder(UserVm userVm, IpAddress publicIp, File pkFile, int nodeSshPort) { + try { + String command = String.format("sudo rm -rf /var/lib/cloud/instances/%s/sem/*", userVm.getUuid()); + Pair result = SshHelper.sshExecute(publicIp.getAddress().addr(), nodeSshPort, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName())); + } + } catch (Exception e) { + logger.error(String.format("Failed to cleanup previous applied userdata on node: %s; This may hamper to addition of the node to the cluster ", userVm.getName()), e); + } + } + + private void revertNetworkRules(Network network, long vmId, int port) { + logger.debug(String.format("Reverting network rules for VM ID %s on network %s", vmId, network.getName())); + FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), port, port); + if (Objects.isNull(network.getVpcId())) { + logger.debug(String.format("Removing firewall rule %s", ruleVO.getId())); + firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } + List pfRules = portForwardingRulesDao.listByVm(vmId); + for (PortForwardingRuleVO pfRule : pfRules) { + logger.debug(String.format("Removing port forwarding rule %s", pfRule.getId())); + rulesService.revokePortForwardingRule(pfRule.getId(), true); + } + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java index 0a399071bf2..cf77a83420a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterDestroyWorker.java @@ -23,6 +23,10 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.bgp.BGPService; +import com.cloud.dc.ASNumberVO; +import com.cloud.dc.DataCenter; +import com.cloud.dc.dao.ASNumberDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -63,6 +67,10 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod protected AccountManager accountManager; @Inject private AnnotationDao annotationDao; + @Inject + private ASNumberDao asNumberDao; + @Inject + private BGPService bgpService; private List clusterVMs; @@ -131,6 +139,7 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod Account owner = accountManager.getAccount(network.getAccountId()); User callerUser = accountManager.getActiveUser(CallContext.current().getCallingUserId()); ReservationContext context = new ReservationContextImpl(null, null, callerUser, owner); + releaseASNumber(kubernetesCluster.getZoneId(), kubernetesCluster.getNetworkId()); boolean networkDestroyed = networkMgr.destroyNetwork(kubernetesCluster.getNetworkId(), context, true); if (!networkDestroyed) { String msg = String.format("Failed to destroy network: %s as part of Kubernetes cluster: %s cleanup", network, kubernetesCluster); @@ -143,6 +152,15 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod } } + private void releaseASNumber(Long zoneId, long networkId) { + DataCenter zone = dataCenterDao.findById(zoneId); + ASNumberVO asNumber = asNumberDao.findByZoneAndNetworkId(zone.getId(), networkId); + if (asNumber != null) { + logger.debug(String.format("Releasing AS number %s from network %s", asNumber.getAsNumber(), networkId)); + bgpService.releaseASNumber(zone.getId(), asNumber.getAsNumber(), true); + } + } + protected void deleteKubernetesClusterIsolatedNetworkRules(Network network, List removedVmIds) throws ManagementServerException { IpAddress publicIp = getNetworkSourceNatIp(network); if (publicIp == null) { @@ -157,7 +175,7 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod if (firewallRule == null) { logMessage(Level.WARN, "Firewall rule for API access can't be removed", null); } - firewallRule = removeSshFirewallRule(publicIp); + firewallRule = removeSshFirewallRule(publicIp, network.getId()); if (firewallRule == null) { logMessage(Level.WARN, "Firewall rule for SSH access can't be removed", null); } @@ -256,6 +274,12 @@ public class KubernetesClusterDestroyWorker extends KubernetesClusterResourceMod } if (cleanupNetwork) { // if network has additional VM, cannot proceed with cluster destroy NetworkVO network = networkDao.findById(kubernetesCluster.getNetworkId()); + List externalNodes = clusterVMs.stream().filter(KubernetesClusterVmMapVO::isExternalNode).collect(Collectors.toList()); + if (!externalNodes.isEmpty()) { + String errMsg = String.format("Failed to delete kubernetes cluster %s as there are %s external node(s) present. Please remove the external node(s) from the cluster (and network) or delete them before deleting the cluster.", kubernetesCluster.getName(), externalNodes.size()); + logger.error(errMsg); + throw new CloudRuntimeException(errMsg); + } if (network != null) { List networkVMs = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(network.getId(), VirtualMachine.Type.User); if (networkVMs.size() > clusterVMs.size()) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java new file mode 100644 index 00000000000..07d062e23a1 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterRemoveWorker.java @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.event.ActionEventUtils; +import com.cloud.event.EventVO; +import com.cloud.exception.ManagementServerException; +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import com.cloud.kubernetes.cluster.KubernetesClusterVO; +import com.cloud.network.IpAddress; +import com.cloud.network.Network; +import com.cloud.network.dao.FirewallRulesDao; +import com.cloud.network.rules.FirewallRuleVO; +import com.cloud.network.rules.PortForwardingRuleVO; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.ssh.SshHelper; +import com.cloud.vm.UserVmVO; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.context.CallContext; + +import javax.inject.Inject; +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +public class KubernetesClusterRemoveWorker extends KubernetesClusterActionWorker { + + @Inject + private FirewallRulesDao firewallRulesDao; + + private long removeNodeTimeoutTime; + + public KubernetesClusterRemoveWorker(KubernetesCluster kubernetesCluster, KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + + public boolean removeNodesFromCluster(List nodeIds) { + init(); + removeNodeTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterRemoveNodeTimeout.value() * 1000; + Long networkId = kubernetesCluster.getNetworkId(); + Network network = networkDao.findById(networkId); + if (Objects.isNull(network)) { + throw new CloudRuntimeException(String.format("Failed to find network with id: %s", networkId)); + } + IpAddress publicIp = null; + try { + publicIp = getPublicIp(network); + } catch (ManagementServerException e) { + throw new CloudRuntimeException(String.format("Failed to retrieve public IP for the network: %s ", network.getName())); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.RemoveNodeRequested); + boolean result = removeNodesFromCluster(nodeIds, network, publicIp); + if (!result) { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); + } else { + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + } + String description = String.format("Successfully removed %s nodes from the Kubernetes Cluster %s", nodeIds.size(), kubernetesCluster.getUuid()); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE, + description, kubernetesCluster.getId(), ApiCommandResourceType.KubernetesCluster.toString(), 0); + return result; + } + + private boolean removeNodesFromCluster(List nodeIds, Network network, IpAddress publicIp) { + boolean result = true; + List removedNodeIds = new ArrayList<>(); + long removedMemory = 0L; + long removedCores = 0L; + for (Long nodeId : nodeIds) { + UserVmVO vm = userVmDao.findById(nodeId); + if (vm == null) { + logger.debug(String.format("Couldn't find a VM with ID %s, skipping removal from Kubernetes cluster", nodeId)); + continue; + } + try { + removeNodeVmFromCluster(nodeId, vm.getDisplayName().toLowerCase(Locale.ROOT), publicIp.getAddress().addr()); + result &= removeNodePortForwardingRules(nodeId, network, vm); + if (System.currentTimeMillis() > removeNodeTimeoutTime) { + logger.error(String.format("Removal of node %s from Kubernetes cluster %s timed out", vm.getName(), kubernetesCluster.getName())); + result = false; + continue; + } + ServiceOfferingVO offeringVO = serviceOfferingDao.findById(vm.getId(), vm.getServiceOfferingId()); + removedNodeIds.add(nodeId); + removedMemory += offeringVO.getRamSize(); + removedCores += offeringVO.getCpu(); + String description = String.format("Successfully removed the node %s from Kubernetes cluster %s", vm.getUuid(), kubernetesCluster.getUuid()); + logger.info(description); + ActionEventUtils.onCompletedActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), + EventVO.LEVEL_INFO, KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE, + description, vm.getId(), ApiCommandResourceType.VirtualMachine.toString(), 0); + } catch (Exception e) { + String err = String.format("Error trying to remove node %s from Kubernetes Cluster %s: %s", vm.getUuid(), kubernetesCluster.getUuid(), e.getMessage()); + logger.error(err, e); + result = false; + } + } + updateKubernetesCluster(kubernetesCluster.getId(), removedNodeIds, removedMemory, removedCores); + return result; + } + + protected boolean removeNodePortForwardingRules(Long nodeId, Network network, UserVmVO vm) { + List pfRules = portForwardingRulesDao.listByVm(nodeId); + boolean result = true; + for (PortForwardingRuleVO pfRule : pfRules) { + try { + result &= rulesService.revokePortForwardingRule(pfRule.getId(), true); + if (Objects.isNull(network.getVpcId())) { + FirewallRuleVO ruleVO = firewallRulesDao.findByNetworkIdAndPorts(network.getId(), pfRule.getSourcePortStart(), pfRule.getSourcePortEnd()); + result &= firewallService.revokeIngressFirewallRule(ruleVO.getId(), true); + } + } catch (Exception e) { + String err = String.format("Failed to cleanup network rules for node %s, due to: %s", vm.getName(), e.getMessage()); + logger.error(err, e); + } + } + return result; + } + + private void removeNodeVmFromCluster(Long nodeId, String nodeName, String publicIp) throws Exception { + File removeNodeScriptFile = retrieveScriptFile(removeNodeFromClusterScript); + copyScriptFile(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, removeNodeScriptFile, removeNodeFromClusterScript); + File pkFile = getManagementServerSshPublicKeyFile(); + String command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "remove"); + Pair result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Node: %s failed to be gracefully drained as a worker node from cluster %s ", nodeName, kubernetesCluster.getName())); + } + List nodePfRules = portForwardingRulesDao.listByVm(nodeId); + Optional nodeSshPort = nodePfRules.stream().filter(rule -> rule.getDestinationPortStart() == DEFAULT_SSH_PORT + && rule.getVirtualMachineId() == nodeId && rule.getSourcePortStart() >= CLUSTER_NODES_DEFAULT_START_SSH_PORT).findFirst(); + if (nodeSshPort.isPresent()) { + copyScriptFile(publicIp, nodeSshPort.get().getSourcePortStart(), removeNodeScriptFile, removeNodeFromClusterScript); + command = String.format("sudo %s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "worker", "remove"); + result = SshHelper.sshExecute(publicIp, nodeSshPort.get().getSourcePortStart(), getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Failed to reset node: %s from cluster %s ", nodeName, kubernetesCluster.getName())); + } + command = String.format("%s%s %s %s %s", scriptPath, removeNodeFromClusterScript, nodeName, "control", "delete"); + result = SshHelper.sshExecute(publicIp, CLUSTER_NODES_DEFAULT_START_SSH_PORT, getControlNodeLoginUser(), + pkFile, null, command, 10000, 10000, 10 * 60 * 1000); + if (Boolean.FALSE.equals(result.first())) { + logger.error(String.format("Node: %s failed to be gracefully delete node from cluster %s ", nodeName, kubernetesCluster.getName())); + } + + } + } + + private void updateKubernetesCluster(long clusterId, List nodesRemoved, long deallocatedRam, long deallocatedCores) { + KubernetesClusterVO kubernetesClusterVO = kubernetesClusterDao.findById(clusterId); + kubernetesClusterVO.setNodeCount(kubernetesClusterVO.getNodeCount() - nodesRemoved.size()); + kubernetesClusterVO.setMemory(kubernetesClusterVO.getMemory() - deallocatedRam); + kubernetesClusterVO.setCores(kubernetesClusterVO.getCores() - deallocatedCores); + kubernetesClusterDao.update(clusterId, kubernetesClusterVO); + + nodesRemoved.forEach(id -> kubernetesClusterVmMapDao.removeByClusterIdAndVmIdsIn(clusterId, nodesRemoved)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index a8c4b307065..7f7f8c07f06 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -17,23 +17,37 @@ package com.cloud.kubernetes.cluster.actionworkers; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; import static com.cloud.utils.NumbersUtil.toHumanReadableSize; +import static com.cloud.utils.db.Transaction.execute; import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.dc.DedicatedResourceVO; +import com.cloud.dc.dao.DedicatedResourceDao; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.network.rules.RulesService; +import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.rules.FirewallManager; import com.cloud.offering.NetworkOffering; import com.cloud.offerings.dao.NetworkOfferingDao; -import org.apache.cloudstack.api.ApiConstants; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.net.Ip; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd; import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd; @@ -64,23 +78,18 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.kubernetes.cluster.KubernetesCluster; -import com.cloud.kubernetes.cluster.KubernetesClusterDetailsVO; import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; import com.cloud.kubernetes.cluster.KubernetesClusterVO; -import com.cloud.kubernetes.cluster.utils.KubernetesClusterUtil; import com.cloud.network.IpAddress; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.dao.LoadBalancerDao; import com.cloud.network.dao.LoadBalancerVO; -import com.cloud.network.firewall.FirewallService; import com.cloud.network.lb.LoadBalancingRulesService; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.rules.LoadBalancer; import com.cloud.network.rules.PortForwardingRuleVO; -import com.cloud.network.rules.RulesService; -import com.cloud.network.rules.dao.PortForwardingRulesDao; import com.cloud.network.vpc.NetworkACL; import com.cloud.network.vpc.NetworkACLItem; import com.cloud.network.vpc.NetworkACLItemDao; @@ -94,15 +103,12 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.LaunchPermissionDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.user.Account; -import com.cloud.user.SSHKeyPairVO; import com.cloud.uservm.UserVm; import com.cloud.utils.Pair; import com.cloud.utils.component.ComponentContext; -import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionCallbackWithException; import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.net.Ip; import com.cloud.utils.net.NetUtils; import com.cloud.utils.ssh.SshHelper; import com.cloud.vm.Nic; @@ -127,8 +133,6 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected FirewallRulesDao firewallRulesDao; @Inject - protected FirewallService firewallService; - @Inject protected NetworkACLService networkACLService; @Inject protected NetworkACLItemDao networkACLItemDao; @@ -143,6 +147,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu @Inject protected ResourceManager resourceManager; @Inject + protected DedicatedResourceDao dedicatedResourceDao; + @Inject protected LoadBalancerDao loadBalancerDao; @Inject protected VMInstanceDao vmInstanceDao; @@ -168,81 +174,37 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu kubernetesClusterNodeNamePrefix = getKubernetesClusterNodeNamePrefix(); } - private String getKubernetesNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sNodeConfig = readResourceFile("/conf/k8s-node.yml"); - final String sshPubKey = "{{ k8s.ssh.pub.key }}"; - final String joinIpKey = "{{ k8s_control_node.join_ip }}"; - final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; - final String ejectIsoKey = "{{ k8s.eject.iso }}"; - - String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; - String sshKeyPair = kubernetesCluster.getKeyPair(); - if (StringUtils.isNotEmpty(sshKeyPair)) { - SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); - if (sshkp != null) { - pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; - } - } - k8sNodeConfig = k8sNodeConfig.replace(sshPubKey, pubKey); - k8sNodeConfig = k8sNodeConfig.replace(joinIpKey, joinIp); - k8sNodeConfig = k8sNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); - k8sNodeConfig = k8sNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); - k8sNodeConfig = updateKubeConfigWithRegistryDetails(k8sNodeConfig); - - return k8sNodeConfig; - } - - protected String updateKubeConfigWithRegistryDetails(String k8sConfig) { - /* genarate /etc/containerd/config.toml file on the nodes only if Kubernetes cluster is created to - * use docker private registry */ - String registryUsername = null; - String registryPassword = null; - String registryUrl = null; - - List details = kubernetesClusterDetailsDao.listDetails(kubernetesCluster.getId()); - for (KubernetesClusterDetailsVO detail : details) { - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_USER_NAME)) { - registryUsername = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_PASSWORD)) { - registryPassword = detail.getValue(); - } - if (detail.getName().equals(ApiConstants.DOCKER_REGISTRY_URL)) { - registryUrl = detail.getValue(); - } - } - - if (StringUtils.isNoneEmpty(registryUsername, registryPassword, registryUrl)) { - // Update runcmd in the cloud-init configuration to run a script that updates the containerd config with provided registry details - String runCmd = "- bash -x /opt/bin/setup-containerd"; - - String registryEp = registryUrl.split("://")[1]; - k8sConfig = k8sConfig.replace("- containerd config default > /etc/containerd/config.toml", runCmd); - final String registryUrlKey = "{{registry.url}}"; - final String registryUrlEpKey = "{{registry.url.endpoint}}"; - final String registryAuthKey = "{{registry.token}}"; - final String registryUname = "{{registry.username}}"; - final String registryPsswd = "{{registry.password}}"; - - final String usernamePasswordKey = registryUsername + ":" + registryPassword; - String base64Auth = Base64.encodeBase64String(usernamePasswordKey.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); - k8sConfig = k8sConfig.replace(registryUrlKey, registryUrl); - k8sConfig = k8sConfig.replace(registryUrlEpKey, registryEp); - k8sConfig = k8sConfig.replace(registryUname, registryUsername); - k8sConfig = k8sConfig.replace(registryPsswd, registryPassword); - k8sConfig = k8sConfig.replace(registryAuthKey, base64Auth); - } - return k8sConfig; - } - protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering) throws InsufficientServerCapacityException { + protected DeployDestination plan(final long nodesCount, final DataCenter zone, final ServiceOffering offering, + final Long domainId, final Long accountId, final Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException { final int cpu_requested = offering.getCpu() * offering.getSpeed(); final long ram_requested = offering.getRamSize() * 1024L * 1024L; - List hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId()); + boolean useDedicatedHosts = false; + List hosts = new ArrayList<>(); + Long group = getExplicitAffinityGroup(domainId, accountId); + if (Objects.nonNull(group)) { + List dedicatedHosts = new ArrayList<>(); + if (Objects.nonNull(accountId)) { + dedicatedHosts = dedicatedResourceDao.listByAccountId(accountId); + } else if (Objects.nonNull(domainId)) { + dedicatedHosts = dedicatedResourceDao.listByDomainId(domainId); + } + for (DedicatedResourceVO dedicatedHost : dedicatedHosts) { + hosts.add(hostDao.findById(dedicatedHost.getHostId())); + useDedicatedHosts = true; + } + } + if (hosts.isEmpty()) { + hosts = resourceManager.listAllHostsInOneZoneByType(Host.Type.Routing, zone.getId()); + } + if (hypervisorType != null) { + hosts = hosts.stream().filter(x -> x.getHypervisorType() == hypervisorType).collect(Collectors.toList()); + } final Map> hosts_with_resevered_capacity = new ConcurrentHashMap>(); for (HostVO h : hosts) { hosts_with_resevered_capacity.put(h.getUuid(), new Pair(h, 0)); } boolean suitable_host_found = false; + HostVO suitableHost = null; for (int i = 1; i <= nodesCount; i++) { suitable_host_found = false; for (Map.Entry> hostEntry : hosts_with_resevered_capacity.entrySet()) { @@ -269,6 +231,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu logger.debug("Found host {} with enough capacity: CPU={} RAM={}", h.getName(), cpu_requested * reserved, toHumanReadableSize(ram_requested * reserved)); hostEntry.setValue(new Pair(h, reserved)); suitable_host_found = true; + suitableHost = h; break; } } @@ -284,6 +247,9 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (logger.isInfoEnabled()) { logger.info("Suitable hosts found in datacenter: {}, creating deployment destination", zone); } + if (useDedicatedHosts) { + return new DeployDestination(zone, null, null, suitableHost); + } return new DeployDestination(zone, null, null, null); } String msg = String.format("Cannot find enough capacity for Kubernetes cluster(requested cpu=%d memory=%s) with offering: %s and hypervisor: %s", @@ -293,13 +259,35 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu throw new InsufficientServerCapacityException(msg, DataCenter.class, zone.getId()); } - protected DeployDestination plan() throws InsufficientServerCapacityException { - ServiceOffering offering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + /** + * Plan Kubernetes Cluster Deployment + * @return a map of DeployDestination per node type + */ + protected Map planKubernetesCluster(Long domainId, Long accountId, Hypervisor.HypervisorType hypervisorType) throws InsufficientServerCapacityException { + Map destinationMap = new HashMap<>(); DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); if (logger.isDebugEnabled()) { logger.debug("Checking deployment destination for Kubernetes cluster: {} in zone: {}", kubernetesCluster, zone); } - return plan(kubernetesCluster.getTotalNodeCount(), zone, offering); + long controlNodeCount = kubernetesCluster.getControlNodeCount(); + long clusterSize = kubernetesCluster.getNodeCount(); + long etcdNodes = kubernetesCluster.getEtcdNodeCount(); + Map nodeTypeCount = Map.of(WORKER.name(), clusterSize, + CONTROL.name(), controlNodeCount, ETCD.name(), etcdNodes); + + for (KubernetesClusterNodeType nodeType : CLUSTER_NODES_TYPES_LIST) { + Long nodes = nodeTypeCount.getOrDefault(nodeType.name(), kubernetesCluster.getServiceOfferingId()); + if (nodes == null || nodes == 0) { + continue; + } + ServiceOffering nodeOffering = getServiceOfferingForNodeTypeOnCluster(nodeType, kubernetesCluster); + if (logger.isDebugEnabled()) { + logger.debug("Checking deployment destination for {} nodes on Kubernetes cluster : {} in zone : {}", nodeType.name(), kubernetesCluster.getName(), zone.getName()); + } + DeployDestination planForNodeType = plan(nodes, zone, nodeOffering, domainId, accountId, hypervisorType); + destinationMap.put(nodeType.name(), planForNodeType); + } + return destinationMap; } protected void resizeNodeVolume(final UserVm vm) throws ManagementServerException { @@ -322,14 +310,33 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } } - protected void startKubernetesVM(final UserVm vm) throws ManagementServerException { + protected void startKubernetesVM(final UserVm vm, final Long domainId, final Long accountId, KubernetesClusterNodeType nodeType) throws ManagementServerException { CallContext vmContext = null; if (!ApiCommandResourceType.VirtualMachine.equals(CallContext.current().getEventResourceType())); { vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine); vmContext.setEventResourceId(vm.getId()); } + DeploymentPlan plan = null; + if (Objects.nonNull(domainId) && !listDedicatedHostsInDomain(domainId).isEmpty()) { + DeployDestination dest = null; + try { + Map destinationMap = planKubernetesCluster(domainId, accountId, vm.getHypervisorType()); + dest = destinationMap.get(nodeType.name()); + } catch (InsufficientCapacityException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + if (dest != null) { + plan = new DataCenterDeployment( + Objects.nonNull(dest.getDataCenter()) ? dest.getDataCenter().getId() : 0, + Objects.nonNull(dest.getPod()) ? dest.getPod().getId() : null, + Objects.nonNull(dest.getCluster()) ? dest.getCluster().getId() : null, + Objects.nonNull(dest.getHost()) ? dest.getHost().getId() : null, + null, + null); + } + } try { - userVmManager.startVirtualMachine(vm); + userVmManager.startVirtualMachine(vm, plan); } catch (OperationTimedoutException | ResourceUnavailableException | InsufficientCapacityException ex) { throw new ManagementServerException(String.format("Failed to start VM in the Kubernetes cluster : %s", kubernetesCluster.getName()), ex); } finally { @@ -344,19 +351,20 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } } - protected List provisionKubernetesClusterNodeVms(final long nodeCount, final int offset, final String publicIpAddress) throws ManagementServerException, + protected List provisionKubernetesClusterNodeVms(final long nodeCount, final int offset, + final String controlIpAddress, final Long domainId, final Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { List nodes = new ArrayList<>(); for (int i = offset + 1; i <= nodeCount; i++) { CallContext vmContext = CallContext.register(CallContext.current(), ApiCommandResourceType.VirtualMachine); try { - UserVm vm = createKubernetesNode(publicIpAddress); + UserVm vm = createKubernetesNode(controlIpAddress, domainId, accountId); vmContext.setEventResourceId(vm.getId()); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } - startKubernetesVM(vm); + startKubernetesVM(vm, domainId, accountId, WORKER); vm = userVmDao.findById(vm.getId()); if (vm == null) { throw new ManagementServerException(String.format("Failed to provision worker VM for Kubernetes cluster : %s", kubernetesCluster.getName())); @@ -370,16 +378,16 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu return nodes; } - protected List provisionKubernetesClusterNodeVms(final long nodeCount, final String publicIpAddress) throws ManagementServerException, + protected List provisionKubernetesClusterNodeVms(final long nodeCount, final String controlIpAddress, final Long domainId, final Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { - return provisionKubernetesClusterNodeVms(nodeCount, 0, publicIpAddress); + return provisionKubernetesClusterNodeVms(nodeCount, 0, controlIpAddress, domainId, accountId); } - protected UserVm createKubernetesNode(String joinIp) throws ManagementServerException, + protected UserVm createKubernetesNode(String joinIp, Long domainId, Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm nodeVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(WORKER, kubernetesCluster); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Account owner = accountDao.findById(kubernetesCluster.getAccountId()); @@ -396,7 +404,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu String hostName = String.format("%s-node-%s", kubernetesClusterNodeNamePrefix, suffix); String k8sNodeConfig = null; try { - k8sNodeConfig = getKubernetesNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sNodeConfig = getKubernetesNodeConfig(joinIp, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), false); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes node configuration file", e); } @@ -406,18 +414,21 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { keypairs.add(kubernetesCluster.getKeyPair()); } + Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) { List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); - nodeVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, securityGroupIds, owner, + nodeVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, securityGroupIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, - null, addrs, null, null, null, customParameterMap, null, null, null, + null, addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE); } else { - nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, + nodeVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, workerNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, - null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); + null, addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); } if (logger.isInfoEnabled()) { logger.info("Created node VM : {}, {} in the Kubernetes cluster : {}", hostName, nodeVm, kubernetesCluster.getName()); @@ -455,7 +466,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu final long domainId = account.getDomainId(); Nic vmNic = networkModel.getNicInNetwork(vmId, networkId); final Ip vmIp = new Ip(vmNic.getIPv4Address()); - PortForwardingRuleVO pfRule = Transaction.execute((TransactionCallbackWithException) status -> { + PortForwardingRuleVO pfRule = execute((TransactionCallbackWithException) status -> { PortForwardingRuleVO newRule = new PortForwardingRuleVO(null, publicIpId, sourcePort, sourcePort, @@ -487,11 +498,18 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu * @throws NetworkRuleConflictException */ protected void provisionSshPortForwardingRules(IpAddress publicIp, Network network, Account account, - List clusterVMIds) throws ResourceUnavailableException, + List clusterVMIds, Map vmIdPortMap) throws ResourceUnavailableException, NetworkRuleConflictException { if (!CollectionUtils.isEmpty(clusterVMIds)) { - for (int i = 0; i < clusterVMIds.size(); ++i) { - provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), CLUSTER_NODES_DEFAULT_START_SSH_PORT + i, DEFAULT_SSH_PORT); + int defaultNodesCount = clusterVMIds.size() - vmIdPortMap.size(); + int sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT; + for (int i = 0; i < defaultNodesCount; ++i) { + sourcePort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + i; + provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT); + } + for (int i = defaultNodesCount; i < clusterVMIds.size(); ++i) { + sourcePort += 1; + provisionPublicIpPortForwardingRule(publicIp, network, account, clusterVMIds.get(i), sourcePort, DEFAULT_SSH_PORT); } } } @@ -513,15 +531,15 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu return rule; } - protected FirewallRule removeSshFirewallRule(final IpAddress publicIp) { + protected FirewallRule removeSshFirewallRule(final IpAddress publicIp, final long networkId) { FirewallRule rule = null; List firewallRules = firewallRulesDao.listByIpPurposeProtocolAndNotRevoked(publicIp.getId(), FirewallRule.Purpose.Firewall, NetUtils.TCP_PROTO); for (FirewallRuleVO firewallRule : firewallRules) { - Integer startPort = firewallRule.getSourcePortStart(); - if (startPort != null && startPort == CLUSTER_NODES_DEFAULT_START_SSH_PORT) { + PortForwardingRuleVO pfRule = portForwardingRulesDao.findByNetworkAndPorts(networkId, firewallRule.getSourcePortStart(), firewallRule.getSourcePortEnd()); + if (firewallRule.getSourcePortStart() == CLUSTER_NODES_DEFAULT_START_SSH_PORT || (Objects.nonNull(pfRule) && pfRule.getDestinationPortStart() == DEFAULT_SSH_PORT) ) { rule = firewallRule; firewallService.revokeIngressFwRule(firewallRule.getId(), true); - logger.debug("The SSH firewall rule [%s] with the id [%s] was revoked",firewallRule.getName(),firewallRule.getId()); + logger.debug("The SSH firewall rule {} with the id {} was revoked", firewallRule.getName(), firewallRule.getId()); break; } } @@ -540,7 +558,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu logger.trace("Marking PF rule {} with Revoke state", pfRule); pfRule.setState(FirewallRule.State.Revoke); revokedRules.add(pfRule); - logger.debug("The Port forwarding rule [%s] with the id [%s] was removed.", pfRule.getName(), pfRule.getId()); + logger.debug("The Port forwarding rule {} with the id {} was removed.", pfRule.getName(), pfRule.getId()); break; } } @@ -633,22 +651,11 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu lbService.assignToLoadBalancer(lb.getId(), null, vmIdIpMap, false); } - protected void createFirewallRules(IpAddress publicIp, List clusterVMIds, boolean apiRule) throws ManagementServerException { + protected Map createFirewallRules(IpAddress publicIp, List clusterVMIds, boolean apiRule) throws ManagementServerException { // Firewall rule for SSH access on each node VM - CallContext.register(CallContext.current(), null); - try { - int endPort = CLUSTER_NODES_DEFAULT_START_SSH_PORT + clusterVMIds.size() - 1; - provisionFirewallRules(publicIp, owner, CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort); - if (logger.isInfoEnabled()) { - logger.info("Provisioned firewall rule to open up port {} to {} on {} for Kubernetes cluster: {}", CLUSTER_NODES_DEFAULT_START_SSH_PORT, endPort, publicIp.getAddress().addr(), kubernetesCluster); - } - } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | NetworkRuleConflictException e) { - throw new ManagementServerException(String.format("Failed to provision firewall rules for SSH access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); - } finally { - CallContext.unregister(); - } + Map vmIdPortMap = addFirewallRulesForNodes(publicIp, clusterVMIds.size()); if (!apiRule) { - return; + return vmIdPortMap; } // Firewall rule for API access for control node VMs CallContext.register(CallContext.current(), null); @@ -662,6 +669,7 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } finally { CallContext.unregister(); } + return vmIdPortMap; } /** @@ -676,11 +684,11 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu * @throws ManagementServerException */ protected void setupKubernetesClusterIsolatedNetworkRules(IpAddress publicIp, Network network, List clusterVMIds, boolean apiRule) throws ManagementServerException { - createFirewallRules(publicIp, clusterVMIds, apiRule); + Map vmIdPortMap = createFirewallRules(publicIp, clusterVMIds, apiRule); // Port forwarding rule for SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -772,7 +780,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu // Add port forwarding rule for SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds); + Map vmIdPortMap = getVmPortMap(); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -793,8 +802,27 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu return prefix; } + protected String getEtcdNodeNameForCluster() { + String prefix = kubernetesCluster.getName(); + if (!NetUtils.verifyDomainNameLabel(prefix, true)) { + prefix = prefix.replaceAll("[^a-zA-Z0-9-]", ""); + if (prefix.isEmpty()) { + prefix = kubernetesCluster.getUuid(); + } + } + prefix = prefix + "-etcd" ; + if (prefix.length() > 40) { + prefix = prefix.substring(0, 40); + } + return prefix; + } + protected KubernetesClusterVO updateKubernetesClusterEntry(final Long cores, final Long memory, final Long size, - final Long serviceOfferingId, final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) { + final Long serviceOfferingId, final Boolean autoscaleEnabled, + final Long minSize, final Long maxSize, + final KubernetesClusterNodeType nodeType, + final boolean updateNodeOffering, + final boolean updateClusterOffering) { return Transaction.execute((TransactionCallback) status -> { KubernetesClusterVO updatedCluster = kubernetesClusterDao.createForUpdate(kubernetesCluster.getId()); @@ -807,7 +835,16 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu if (size != null) { updatedCluster.setNodeCount(size); } - if (serviceOfferingId != null) { + if (updateNodeOffering && serviceOfferingId != null && nodeType != null) { + if (WORKER == nodeType) { + updatedCluster.setWorkerNodeServiceOfferingId(serviceOfferingId); + } else if (CONTROL == nodeType) { + updatedCluster.setControlNodeServiceOfferingId(serviceOfferingId); + } else if (ETCD == nodeType) { + updatedCluster.setEtcdNodeServiceOfferingId(serviceOfferingId); + } + } + if (updateClusterOffering && serviceOfferingId != null) { updatedCluster.setServiceOfferingId(serviceOfferingId); } if (autoscaleEnabled != null) { @@ -815,12 +852,14 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu } updatedCluster.setMinSize(minSize); updatedCluster.setMaxSize(maxSize); - return kubernetesClusterDao.persist(updatedCluster); + kubernetesClusterDao.persist(updatedCluster); + // Prevent null attributes set by the createForUpdate method + return kubernetesClusterDao.findById(kubernetesCluster.getId()); }); } private KubernetesClusterVO updateKubernetesClusterEntry(final Boolean autoscaleEnabled, final Long minSize, final Long maxSize) throws CloudRuntimeException { - KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize); + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(null, null, null, null, autoscaleEnabled, minSize, maxSize, null, false, false); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); @@ -883,4 +922,8 @@ public class KubernetesClusterResourceModifierActionWorker extends KubernetesClu updateLoginUserDetails(null); } } + + protected List listDedicatedHostsInDomain(Long domainId) { + return dedicatedResourceDao.listByDomainId(domainId); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java index 4d50ef7e1f8..bfc553f6afa 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorker.java @@ -19,17 +19,23 @@ package com.cloud.kubernetes.cluster.actionworkers; import java.io.File; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.VMTemplateVO; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.context.CallContext; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.dc.DataCenter; @@ -60,12 +66,17 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; import org.apache.logging.log4j.Level; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModifierActionWorker { @Inject protected VMInstanceDao vmInstanceDao; - private ServiceOffering serviceOffering; + private Map serviceOfferingNodeTypeMap; private Long clusterSize; private List nodeIds; private KubernetesCluster.State originalState; @@ -75,8 +86,12 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif private Boolean isAutoscalingEnabled; private long scaleTimeoutTime; + protected KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) { + super(kubernetesCluster, clusterManager); + } + public KubernetesClusterScaleWorker(final KubernetesCluster kubernetesCluster, - final ServiceOffering serviceOffering, + final Map serviceOfferingNodeTypeMap, final Long clusterSize, final List nodeIds, final Boolean isAutoscalingEnabled, @@ -84,7 +99,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif final Long maxSize, final KubernetesClusterManagerImpl clusterManager) { super(kubernetesCluster, clusterManager); - this.serviceOffering = serviceOffering; + this.serviceOfferingNodeTypeMap = serviceOfferingNodeTypeMap; this.nodeIds = nodeIds; this.isAutoscalingEnabled = isAutoscalingEnabled; this.minSize = minSize; @@ -123,7 +138,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } // Remove existing SSH firewall rules - FirewallRule firewallRule = removeSshFirewallRule(publicIp); + FirewallRule firewallRule = removeSshFirewallRule(publicIp, network.getId()); if (firewallRule == null) { throw new ManagementServerException("Firewall rule for node SSH access can't be provisioned"); } @@ -148,7 +163,8 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } // Add port forwarding rule for SSH access on each node VM try { - provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds); + Map vmIdPortMap = getVmPortMap(); + provisionSshPortForwardingRules(publicIp, network, owner, clusterVMIds, vmIdPortMap); } catch (ResourceUnavailableException | NetworkRuleConflictException e) { throw new ManagementServerException(String.format("Failed to activate SSH port forwarding rules for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); } @@ -176,15 +192,19 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif scaleKubernetesClusterIsolatedNetworkRules(clusterVMIds); } - private KubernetesClusterVO updateKubernetesClusterEntry(final Long newSize, final ServiceOffering newServiceOffering) throws CloudRuntimeException { + private KubernetesClusterVO updateKubernetesClusterEntryForNodeType(final Long newWorkerSize, final KubernetesClusterNodeType nodeType, + final ServiceOffering newServiceOffering, + final boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException { final ServiceOffering serviceOffering = newServiceOffering == null ? serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()) : newServiceOffering; final Long serviceOfferingId = newServiceOffering == null ? null : serviceOffering.getId(); - final long size = newSize == null ? kubernetesCluster.getTotalNodeCount() : (newSize + kubernetesCluster.getControlNodeCount()); - final long cores = serviceOffering.getCpu() * size; - final long memory = serviceOffering.getRamSize() * size; - KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newSize, serviceOfferingId, - kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize()); + + Pair clusterCountAndCapacity = calculateNewClusterCountAndCapacity(newWorkerSize, nodeType, serviceOffering); + long cores = clusterCountAndCapacity.first(); + long memory = clusterCountAndCapacity.second(); + + KubernetesClusterVO kubernetesClusterVO = updateKubernetesClusterEntry(cores, memory, newWorkerSize, serviceOfferingId, + kubernetesCluster.getAutoscalingEnabled(), kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize(), nodeType, updateNodeOffering, updateClusterOffering); if (kubernetesClusterVO == null) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, unable to update Kubernetes cluster", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); @@ -192,6 +212,58 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif return kubernetesClusterVO; } + protected Pair calculateNewClusterCountAndCapacity(Long newWorkerSize, KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering) { + long cores; + long memory; + long totalClusterSize = newWorkerSize == null ? kubernetesCluster.getTotalNodeCount() : (newWorkerSize + kubernetesCluster.getControlNodeCount() + kubernetesCluster.getEtcdNodeCount()); + + if (nodeType == DEFAULT) { + cores = serviceOffering.getCpu() * totalClusterSize; + memory = serviceOffering.getRamSize() * totalClusterSize; + } else { + long nodeCount = getNodeCountForType(nodeType, kubernetesCluster); + Long existingOfferingId = getExistingOfferingIdForNodeType(nodeType, kubernetesCluster); + if (existingOfferingId == null) { + existingOfferingId = serviceOffering.getId(); + } + ServiceOfferingVO previousOffering = serviceOfferingDao.findById(existingOfferingId); + Pair previousNodesCapacity = calculateNodesCapacity(previousOffering, nodeCount); + if (WORKER == nodeType) { + nodeCount = newWorkerSize == null ? kubernetesCluster.getNodeCount() : newWorkerSize; + } + Pair newNodesCapacity = calculateNodesCapacity(serviceOffering, nodeCount); + Pair newClusterCapacity = calculateClusterNewCapacity(kubernetesCluster, previousNodesCapacity, newNodesCapacity); + cores = newClusterCapacity.first(); + memory = newClusterCapacity.second(); + } + return new Pair<>(cores, memory); + } + + private long getNodeCountForType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { + if (WORKER == nodeType) { + return kubernetesCluster.getNodeCount(); + } else if (CONTROL == nodeType) { + return kubernetesCluster.getControlNodeCount(); + } else if (ETCD == nodeType) { + return kubernetesCluster.getEtcdNodeCount(); + } + return kubernetesCluster.getTotalNodeCount(); + } + + protected Pair calculateClusterNewCapacity(KubernetesCluster kubernetesCluster, + Pair previousNodeTypeCapacity, + Pair newNodeTypeCapacity) { + long previousCores = kubernetesCluster.getCores(); + long previousMemory = kubernetesCluster.getMemory(); + long newCores = previousCores - previousNodeTypeCapacity.first() + newNodeTypeCapacity.first(); + long newMemory = previousMemory - previousNodeTypeCapacity.second() + newNodeTypeCapacity.second(); + return new Pair<>(newCores, newMemory); + } + + protected Pair calculateNodesCapacity(ServiceOffering offering, long nodeCount) { + return new Pair<>(offering.getCpu() * nodeCount, offering.getRamSize() * nodeCount); + } + private boolean removeKubernetesClusterNode(final String ipAddress, final int port, final UserVm userVm, final int retries, final int waitDuration) { File pkFile = getManagementServerSshPublicKeyFile(); int retryCounter = 0; @@ -266,11 +338,12 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } if (newVmRequiredCount > 0) { final DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); try { if (originalState.equals(KubernetesCluster.State.Running)) { - plan(newVmRequiredCount, zone, clusterServiceOffering); + plan(newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType()); } else { - plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering); + plan(kubernetesCluster.getTotalNodeCount() + newVmRequiredCount, zone, clusterServiceOffering, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId(), clusterTemplate.getHypervisorType()); } } catch (InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.WARN, String.format("Scaling failed for Kubernetes cluster : %s in zone : %s, insufficient capacity", kubernetesCluster.getName(), zone.getName())); @@ -282,17 +355,18 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } } - private void scaleKubernetesClusterOffering() throws CloudRuntimeException { + private void scaleKubernetesClusterOffering(KubernetesClusterNodeType nodeType, ServiceOffering serviceOffering, + boolean updateNodeOffering, boolean updateClusterOffering) throws CloudRuntimeException { validateKubernetesClusterScaleOfferingParameters(); if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); } if (KubernetesCluster.State.Created.equals(originalState)) { - kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); + kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); return; } - final long size = kubernetesCluster.getTotalNodeCount(); - List vmList = kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId()); + final long size = getNodeCountForType(nodeType, kubernetesCluster); + List vmList = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType); final long tobeScaledVMCount = Math.min(vmList.size(), size); for (long i = 0; i < tobeScaledVMCount; i++) { KubernetesClusterVmMapVO vmMapVO = vmList.get((int) i); @@ -310,7 +384,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif logTransitStateAndThrow(Level.WARN, String.format("Scaling Kubernetes cluster : %s failed, scaling action timed out", kubernetesCluster.getName()),kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } } - kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); + kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); } private void removeNodesFromCluster(List vmMaps) throws CloudRuntimeException { @@ -346,7 +420,10 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif // Scale network rules to update firewall rule try { - List clusterVMIds = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List clusterVMIds = getKubernetesClusterVMMaps() + .stream() + .filter(x -> !x.isEtcdNode()) + .map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); scaleKubernetesClusterNetworkRules(clusterVMIds); } catch (ManagementServerException e) { logTransitStateAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes " + @@ -361,10 +438,13 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } List vmList; if (this.nodeIds != null) { - vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds); + vmList = getKubernetesClusterVMMapsForNodes(this.nodeIds).stream().filter(vm -> !vm.isExternalNode()).collect(Collectors.toList()); } else { vmList = getKubernetesClusterVMMaps(); - vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize), vmList.size()); + vmList = vmList.stream() + .filter(vm -> !vm.isExternalNode() && !vm.isControlNode() && !vm.isEtcdNode()) + .collect(Collectors.toList()); + vmList = vmList.subList((int) (kubernetesCluster.getControlNodeCount() + clusterSize - 1), vmList.size()); } Collections.reverse(vmList); removeNodesFromCluster(vmList); @@ -375,16 +455,20 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.ScaleUpRequested); } List clusterVMs = new ArrayList<>(); - LaunchPermissionVO launchPermission = new LaunchPermissionVO(clusterTemplate.getId(), owner.getId()); - launchPermissionDao.persist(launchPermission); + if (isDefaultTemplateUsed()) { + LaunchPermissionVO launchPermission = new LaunchPermissionVO(clusterTemplate.getId(), owner.getId()); + launchPermissionDao.persist(launchPermission); + } try { - clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress); + clusterVMs = provisionKubernetesClusterNodeVms((int)(newVmCount + kubernetesCluster.getNodeCount()), (int)kubernetesCluster.getNodeCount(), publicIpAddress, kubernetesCluster.getDomainId(), kubernetesCluster.getAccountId()); updateLoginUserDetails(clusterVMs.stream().map(InternalIdentity::getId).collect(Collectors.toList())); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to provision node VM in the cluster", kubernetesCluster.getName()), e); } try { - List clusterVMIds = getKubernetesClusterVMMaps().stream().map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List externalNodeIds = getKubernetesClusterVMMaps().stream().filter(KubernetesClusterVmMapVO::isExternalNode).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + List clusterVMIds = getKubernetesClusterVMMaps().stream().filter(vm -> !vm.isExternalNode() && !vm.isEtcdNode()).map(KubernetesClusterVmMapVO::getVmId).collect(Collectors.toList()); + clusterVMIds.addAll(externalNodeIds); scaleKubernetesClusterNetworkRules(clusterVMIds); } catch (ManagementServerException e) { logTransitStateToFailedIfNeededAndThrow(Level.ERROR, String.format("Scaling failed for Kubernetes cluster : %s, unable to update network rules", kubernetesCluster.getName()), e); @@ -401,7 +485,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } } - private void scaleKubernetesClusterSize() throws CloudRuntimeException { + private void scaleKubernetesClusterSize(KubernetesClusterNodeType nodeType) throws CloudRuntimeException { validateKubernetesClusterScaleSizeParameters(); final long originalClusterSize = kubernetesCluster.getNodeCount(); final long newVmRequiredCount = clusterSize - originalClusterSize; @@ -409,7 +493,7 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif if (!kubernetesCluster.getState().equals(KubernetesCluster.State.Scaling)) { stateTransitTo(kubernetesCluster.getId(), newVmRequiredCount > 0 ? KubernetesCluster.Event.ScaleUpRequested : KubernetesCluster.Event.ScaleDownRequested); } - kubernetesCluster = updateKubernetesClusterEntry(null, serviceOffering); + kubernetesCluster = updateKubernetesClusterEntryForNodeType(null, nodeType, serviceOfferingNodeTypeMap.get(nodeType.name()), false, false); return; } Pair publicIpSshPort = getKubernetesClusterServerIpSshPort(null); @@ -423,7 +507,9 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } else { // upscale, same node count handled above scaleUpKubernetesClusterSize(newVmRequiredCount); } - kubernetesCluster = updateKubernetesClusterEntry(clusterSize, null); + boolean updateNodeOffering = serviceOfferingNodeTypeMap.containsKey(nodeType.name()); + ServiceOffering nodeOffering = serviceOfferingNodeTypeMap.getOrDefault(nodeType.name(), null); + kubernetesCluster = updateKubernetesClusterEntryForNodeType(clusterSize, nodeType, nodeOffering, updateNodeOffering, false); } private boolean isAutoscalingChanged() { @@ -446,37 +532,99 @@ public class KubernetesClusterScaleWorker extends KubernetesClusterResourceModif } scaleTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterScaleTimeout.value() * 1000; final long originalClusterSize = kubernetesCluster.getNodeCount(); - final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); - if (existingServiceOffering == null) { - logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster)); + boolean scaleClusterDefaultOffering = serviceOfferingNodeTypeMap.containsKey(DEFAULT.name()); + if (scaleClusterDefaultOffering) { + final ServiceOffering existingServiceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + final ServiceOffering existingControlOffering = serviceOfferingDao.findById(kubernetesCluster.getControlNodeServiceOfferingId()); + final ServiceOffering existingWorkerOffering = serviceOfferingDao.findById(kubernetesCluster.getWorkerNodeServiceOfferingId()); + if (existingServiceOffering == null && ObjectUtils.anyNull(existingControlOffering, existingWorkerOffering)) { + logAndThrow(Level.ERROR, String.format("Scaling Kubernetes cluster : %s failed, service offering for the Kubernetes cluster not found!", kubernetesCluster.getName())); + } } - final boolean autoscalingChanged = isAutoscalingChanged(); - final boolean serviceOfferingScalingNeeded = serviceOffering != null && serviceOffering.getId() != existingServiceOffering.getId(); - if (autoscalingChanged) { - boolean autoScaled = autoscaleCluster(this.isAutoscalingEnabled, minSize, maxSize); - if (autoScaled && serviceOfferingScalingNeeded) { - scaleKubernetesClusterOffering(); + final boolean autoscalingChanged = isAutoscalingChanged(); + ServiceOffering defaultServiceOffering = serviceOfferingNodeTypeMap.getOrDefault(DEFAULT.name(), null); + + for (KubernetesClusterNodeType nodeType : Arrays.asList(CONTROL, ETCD, WORKER)) { + boolean isWorkerNodeOrAllNodes = WORKER == nodeType; + final long newVMRequired = (!isWorkerNodeOrAllNodes || clusterSize == null) ? 0 : clusterSize - originalClusterSize; + if (!scaleClusterDefaultOffering && !serviceOfferingNodeTypeMap.containsKey(nodeType.name()) && newVMRequired == 0) { + continue; } - stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); - return autoScaled; - } - final boolean clusterSizeScalingNeeded = clusterSize != null && clusterSize != originalClusterSize; - final long newVMRequired = clusterSize == null ? 0 : clusterSize - originalClusterSize; - if (serviceOfferingScalingNeeded && clusterSizeScalingNeeded) { - if (newVMRequired > 0) { - scaleKubernetesClusterOffering(); - scaleKubernetesClusterSize(); - } else { - scaleKubernetesClusterSize(); - scaleKubernetesClusterOffering(); + + Long existingNodeTypeOfferingId = getKubernetesClusterNodeTypeOfferingId(kubernetesCluster, nodeType); + boolean clusterHasExistingOfferingForNodeType = existingNodeTypeOfferingId != null; + boolean serviceOfferingScalingNeeded = isServiceOfferingScalingNeededForNodeType(nodeType, serviceOfferingNodeTypeMap, kubernetesCluster); + ServiceOffering serviceOffering = serviceOfferingNodeTypeMap.getOrDefault(nodeType.name(), defaultServiceOffering); + boolean updateNodeOffering = serviceOfferingNodeTypeMap.containsKey(nodeType.name()) || + scaleClusterDefaultOffering && clusterHasExistingOfferingForNodeType; + boolean updateClusterOffering = isWorkerNodeOrAllNodes && scaleClusterDefaultOffering; + if (isWorkerNodeOrAllNodes && autoscalingChanged) { + boolean autoScaled = autoscaleCluster(this.isAutoscalingEnabled, minSize, maxSize); + if (autoScaled && serviceOfferingScalingNeeded) { + scaleKubernetesClusterOffering(nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); + return autoScaled; + } + final boolean clusterSizeScalingNeeded = isWorkerNodeOrAllNodes && clusterSize != null && clusterSize != originalClusterSize; + if (serviceOfferingScalingNeeded && clusterSizeScalingNeeded) { + if (newVMRequired > 0) { + scaleKubernetesClusterOffering(nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + scaleKubernetesClusterSize(nodeType); + } else { + scaleKubernetesClusterSize(nodeType); + scaleKubernetesClusterOffering(nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + } + } else if (serviceOfferingScalingNeeded) { + scaleKubernetesClusterOffering(nodeType, serviceOffering, updateNodeOffering, updateClusterOffering); + } else if (clusterSizeScalingNeeded) { + scaleKubernetesClusterSize(nodeType); } - } else if (serviceOfferingScalingNeeded) { - scaleKubernetesClusterOffering(); - } else if (clusterSizeScalingNeeded) { - scaleKubernetesClusterSize(); } + stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); return true; } + + private Long getKubernetesClusterNodeTypeOfferingId(KubernetesCluster kubernetesCluster, KubernetesClusterNodeType nodeType) { + if (nodeType == WORKER) { + return kubernetesCluster.getWorkerNodeServiceOfferingId(); + } else if (nodeType == ETCD) { + return kubernetesCluster.getEtcdNodeServiceOfferingId(); + } else if (nodeType == CONTROL) { + return kubernetesCluster.getControlNodeServiceOfferingId(); + } + return null; + } + + protected boolean isServiceOfferingScalingNeededForNodeType(KubernetesClusterNodeType nodeType, + Map map, KubernetesCluster kubernetesCluster) { + // DEFAULT node type means only the global service offering has been set for the Kubernetes cluster + Long existingOfferingId = map.containsKey(DEFAULT.name()) ? + kubernetesCluster.getServiceOfferingId() : + getExistingOfferingIdForNodeType(nodeType, kubernetesCluster); + if (existingOfferingId == null) { + logAndThrow(Level.ERROR, String.format("The Kubernetes cluster %s does not have a global service offering set", kubernetesCluster.getName())); + } + ServiceOffering existingOffering = serviceOfferingDao.findById(existingOfferingId); + if (existingOffering == null) { + logAndThrow(Level.ERROR, String.format("Cannot find the global service offering with ID %s set on the Kubernetes cluster %s", existingOfferingId, kubernetesCluster.getName())); + } + ServiceOffering newOffering = map.containsKey(DEFAULT.name()) ? map.get(DEFAULT.name()) : map.get(nodeType.name()); + return newOffering != null && newOffering.getId() != existingOffering.getId(); + } + + protected Long getExistingOfferingIdForNodeType(KubernetesClusterNodeType nodeType, KubernetesCluster kubernetesCluster) { + List clusterVms = kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesCluster.getId(), nodeType); + if (CollectionUtils.isEmpty(clusterVms)) { + return null; + } + KubernetesClusterVmMapVO clusterVm = clusterVms.get(0); + UserVmVO clusterUserVm = userVmDao.findById(clusterVm.getVmId()); + if (clusterUserVm == null) { + return null; + } + return clusterUserVm.getServiceOfferingId(); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java index a2384a2e0fe..68bec58d462 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java @@ -24,11 +24,20 @@ import java.net.URL; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.PermissionDeniedException; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; +import com.cloud.network.vpc.NetworkACL; +import com.cloud.storage.VMTemplateVO; +import com.cloud.user.UserDataVO; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.InternalIdentity; import org.apache.cloudstack.framework.ca.Certificate; @@ -75,6 +84,10 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.VmDetailConstants; import org.apache.logging.log4j.Level; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + public class KubernetesClusterStartWorker extends KubernetesClusterResourceModifierActionWorker { private KubernetesSupportedVersion kubernetesClusterVersion; @@ -128,10 +141,10 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return haSupported; } - private String getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, - final String hostName, final boolean haSupported, - final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node.yml"); + private Pair getKubernetesControlNodeConfig(final String controlNodeIp, final String serverIp, + final List etcdIps, final String hostName, final boolean haSupported, + final boolean ejectIso, final boolean externalCni) throws IOException { + String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node.yml"); final String apiServerCert = "{{ k8s_control_node.apiserver.crt }}"; final String apiServerKey = "{{ k8s_control_node.apiserver.key }}"; final String caCert = "{{ k8s_control_node.ca.crt }}"; @@ -139,18 +152,33 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif final String clusterToken = "{{ k8s_control_node.cluster.token }}"; final String clusterInitArgsKey = "{{ k8s_control_node.cluster.initargs }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; + final String installWaitTime = "{{ k8s.install.wait.time }}"; + final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String externalEtcdNodes = "{{ etcd.unstacked_etcd }}"; + final String etcdEndpointList = "{{ etcd.etcd_endpoint_list }}"; + final String k8sServerIp = "{{ k8s_control.server_ip }}"; + final String k8sApiPort = "{{ k8s.api_server_port }}"; + final String certSans = "{{ k8s_control.server_ips }}"; + final String k8sCertificate = "{{ k8s_control.certificate_key }}"; + final String externalCniPlugin = "{{ k8s.external.cni.plugin }}"; final List addresses = new ArrayList<>(); addresses.add(controlNodeIp); if (!serverIp.equals(controlNodeIp)) { addresses.add(serverIp); } + + boolean externalEtcd = !etcdIps.isEmpty(); final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(hostName, "kubernetes", - "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), + "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local"), addresses, 3650, null); final String tlsClientCert = CertUtils.x509CertificateToPem(certificate.getClientCertificate()); final String tlsPrivateKey = CertUtils.privateKeyToPem(certificate.getPrivateKey()); final String tlsCaCert = CertUtils.x509CertificatesToPem(certificate.getCaCertificates()); + final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); + final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); + String endpointList = getEtcdEndpointList(etcdIps); + k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerCert, tlsClientCert.replace("\n", "\n ")); k8sControlNodeConfig = k8sControlNodeConfig.replace(apiServerKey, tlsPrivateKey.replace("\n", "\n ")); k8sControlNodeConfig = k8sControlNodeConfig.replace(caCert, tlsCaCert.replace("\n", "\n ")); @@ -162,29 +190,39 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } + k8sControlNodeConfig = k8sControlNodeConfig.replace(installWaitTime, String.valueOf(waitTime)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterToken, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(externalEtcdNodes, String.valueOf(externalEtcd)); String initArgs = ""; if (haSupported) { initArgs = String.format("--control-plane-endpoint %s:%d --upload-certs --certificate-key %s ", - serverIp, + controlNodeIp, CLUSTER_API_PORT, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); } - initArgs += String.format("--apiserver-cert-extra-sans=%s", serverIp); + initArgs += String.format("--apiserver-cert-extra-sans=%s", controlNodeIp); initArgs += String.format(" --kubernetes-version=%s", getKubernetesClusterVersion().getSemanticVersion()); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterInitArgsKey, initArgs); k8sControlNodeConfig = k8sControlNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(etcdEndpointList, endpointList); + k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sServerIp, controlNodeIp); + k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sApiPort, String.valueOf(CLUSTER_API_PORT)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(certSans, String.format("- %s", serverIp)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(k8sCertificate, KubernetesClusterUtil.generateClusterHACertificateKey(kubernetesCluster)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(externalCniPlugin, String.valueOf(externalCni)); + k8sControlNodeConfig = updateKubeConfigWithRegistryDetails(k8sControlNodeConfig); - return k8sControlNodeConfig; + return new Pair<>(k8sControlNodeConfig, controlNodeIp); } - private UserVm createKubernetesControlNode(final Network network, String serverIp) throws ManagementServerException, + private Pair createKubernetesControlNode(final Network network, String serverIp, List etcdIps, Long domainId, Long accountId, Long asNumber) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm controlVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(CONTROL, kubernetesCluster); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Pair> ipAddresses = getKubernetesControlNodeIpAddresses(zone, network, owner); @@ -205,45 +243,75 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif String suffix = Long.toHexString(System.currentTimeMillis()); String hostName = String.format("%s-control-%s", kubernetesClusterNodeNamePrefix, suffix); boolean haSupported = isKubernetesVersionSupportsHA(); - String k8sControlNodeConfig = null; + Long userDataId = kubernetesCluster.getCniConfigId(); + Pair k8sControlNodeConfigAndControlIp = new Pair<>(null, null); try { - k8sControlNodeConfig = getKubernetesControlNodeConfig(controlNodeIp, serverIp, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + k8sControlNodeConfigAndControlIp = getKubernetesControlNodeConfig(controlNodeIp, serverIp, etcdIps, hostName, haSupported, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType()), Objects.nonNull(userDataId)); } catch (IOException e) { logAndThrow(Level.ERROR, "Failed to read Kubernetes control node configuration file", e); } + String k8sControlNodeConfig = k8sControlNodeConfigAndControlIp.first(); String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + if (Objects.nonNull(userDataId)) { + logger.info("concatenating userdata"); + UserDataVO cniConfigVo = userDataDao.findById(userDataId); + String cniConfig = new String(Base64.decodeBase64(cniConfigVo.getUserData())); + if (Objects.nonNull(asNumber)) { + cniConfig = substituteASNumber(cniConfig, asNumber); + } + cniConfig = Base64.encodeBase64String(cniConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + base64UserData = userDataManager.concatenateUserData(base64UserData, cniConfig, null); + } + List keypairs = new ArrayList(); if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { keypairs.add(kubernetesCluster.getKeyPair()); } + + Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + String userDataDetails = kubernetesCluster.getCniConfigDetails(); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) { List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); - controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, securityGroupIds, owner, - hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, - requestedIps, addrs, null, null, null, customParameterMap, null, null, null, + controlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, + hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, userDataId, userDataDetails, keypairs, + requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE); } else { - controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, + controlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, - Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, - requestedIps, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); + Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, userDataId, userDataDetails, keypairs, + requestedIps, addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); } if (logger.isInfoEnabled()) { logger.info("Created control VM: {}, {} in the Kubernetes cluster: {}", controlVm, hostName, kubernetesCluster); } - return controlVm; + return new Pair<>(controlVm, k8sControlNodeConfigAndControlIp.second()); + } + + private String substituteASNumber(String cniConfig, Long asNumber) { + final String asNumberKey = "{{ AS_NUMBER }}"; + cniConfig = cniConfig.replace(asNumberKey, String.valueOf(asNumber)); + return cniConfig; + } private String getKubernetesAdditionalControlNodeConfig(final String joinIp, final boolean ejectIso) throws IOException { - String k8sControlNodeConfig = readResourceFile("/conf/k8s-control-node-add.yml"); + String k8sControlNodeConfig = readK8sConfigFile("/conf/k8s-control-node-add.yml"); final String joinIpKey = "{{ k8s_control_node.join_ip }}"; final String clusterTokenKey = "{{ k8s_control_node.cluster.token }}"; final String sshPubKey = "{{ k8s.ssh.pub.key }}"; final String clusterHACertificateKey = "{{ k8s_control_node.cluster.ha.certificate.key }}"; final String ejectIsoKey = "{{ k8s.eject.iso }}"; + final String installWaitTime = "{{ k8s.install.wait.time }}"; + final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + + final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); + final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; String sshKeyPair = kubernetesCluster.getKeyPair(); @@ -253,6 +321,8 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; } } + k8sControlNodeConfig = k8sControlNodeConfig.replace(installWaitTime, String.valueOf(waitTime)); + k8sControlNodeConfig = k8sControlNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); k8sControlNodeConfig = k8sControlNodeConfig.replace(sshPubKey, pubKey); k8sControlNodeConfig = k8sControlNodeConfig.replace(joinIpKey, joinIp); k8sControlNodeConfig = k8sControlNodeConfig.replace(clusterTokenKey, KubernetesClusterUtil.generateClusterToken(kubernetesCluster)); @@ -263,11 +333,84 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return k8sControlNodeConfig; } - private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance) throws ManagementServerException, + private String getInitialEtcdClusterDetails(List ipAddresses, List hostnames) { + String initialCluster = "%s=http://%s:%s"; + StringBuilder clusterInfo = new StringBuilder(); + for (int i = 0; i < ipAddresses.size(); i++) { + clusterInfo.append(String.format(initialCluster, hostnames.get(i), ipAddresses.get(i), KubernetesClusterActionWorker.ETCD_NODE_PEER_COMM_PORT)); + if (i < ipAddresses.size()-1) { + clusterInfo.append(","); + } + } + return clusterInfo.toString(); + } + + /** + * + * @param ipAddresses list of etcd node guest IPs + * @return a formatted list of etcd endpoints adhering to YAML syntax + */ + private String getEtcdEndpointList(List ipAddresses) { + StringBuilder endpoints = new StringBuilder(); + for (int i = 0; i < ipAddresses.size(); i++) { + endpoints.append(String.format("- http://%s:%s", ipAddresses.get(i).getIp4Address(), KubernetesClusterActionWorker.ETCD_NODE_CLIENT_REQUEST_PORT)); + if (i < ipAddresses.size()-1) { + endpoints.append("\n "); + } + } + return endpoints.toString(); + } + + + private List getEtcdNodeHostnames() { + List hostnames = new ArrayList<>(); + for (int etcdNodeIndex = 1; etcdNodeIndex <= kubernetesCluster.getEtcdNodeCount(); etcdNodeIndex++) { + String suffix = Long.toHexString(System.currentTimeMillis()); + hostnames.add(String.format("%s-%s-%s", getEtcdNodeNameForCluster(), etcdNodeIndex, suffix)); + } + return hostnames; + } + + private String getEtcdNodeConfig(final List ipAddresses, final List hostnames, final int etcdNodeIndex, + final boolean ejectIso) throws IOException { + String k8sEtcdNodeConfig = readK8sConfigFile("/conf/etcd-node.yml"); + final String sshPubKey = "{{ k8s.ssh.pub.key }}"; + final String ejectIsoKey = "{{ k8s.eject.iso }}"; + final String installWaitTime = "{{ k8s.install.wait.time }}"; + final String installReattemptsCount = "{{ k8s.install.reattempts.count }}"; + final String etcdNodeName = "{{ etcd.node_name }}"; + final String etcdNodeIp = "{{ etcd.node_ip }}"; + final String etcdInitialClusterNodes = "{{ etcd.initial_cluster_nodes }}"; + + final Long waitTime = KubernetesClusterService.KubernetesControlNodeInstallAttemptWait.value(); + final Long reattempts = KubernetesClusterService.KubernetesControlNodeInstallReattempts.value(); + String pubKey = "- \"" + configurationDao.getValue("ssh.publickey") + "\""; + String sshKeyPair = kubernetesCluster.getKeyPair(); + if (StringUtils.isNotEmpty(sshKeyPair)) { + SSHKeyPairVO sshkp = sshKeyPairDao.findByName(owner.getAccountId(), owner.getDomainId(), sshKeyPair); + if (sshkp != null) { + pubKey += "\n - \"" + sshkp.getPublicKey() + "\""; + } + } + String initialClusterDetails = getInitialEtcdClusterDetails(ipAddresses, hostnames); + + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(installWaitTime, String.valueOf(waitTime)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(installReattemptsCount, String.valueOf(reattempts)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(sshPubKey, pubKey); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(ejectIsoKey, String.valueOf(ejectIso)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdNodeName, hostnames.get(etcdNodeIndex)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdNodeIp, ipAddresses.get(etcdNodeIndex)); + k8sEtcdNodeConfig = k8sEtcdNodeConfig.replace(etcdInitialClusterNodes, initialClusterDetails); + + return k8sEtcdNodeConfig; + } + + private UserVm createKubernetesAdditionalControlNode(final String joinIp, final int additionalControlNodeInstance, + final Long domainId, final Long accountId) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { UserVm additionalControlVm = null; DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); - ServiceOffering serviceOffering = serviceOfferingDao.findById(kubernetesCluster.getServiceOfferingId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(CONTROL, kubernetesCluster); List networkIds = new ArrayList(); networkIds.add(kubernetesCluster.getNetworkId()); Network.IpAddresses addrs = new Network.IpAddresses(null, null); @@ -293,20 +436,24 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { keypairs.add(kubernetesCluster.getKeyPair()); } + + Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); if (kubernetesCluster.getSecurityGroupId() != null && networkModel.checkSecurityGroupSupportForNetwork(owner, zone, networkIds, List.of(kubernetesCluster.getSecurityGroupId()))) { List securityGroupIds = new ArrayList<>(); securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); - additionalControlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, securityGroupIds, owner, + additionalControlVm = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, securityGroupIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, - null, addrs, null, null, null, customParameterMap, null, null, null, + null, addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, null, UserVmManager.CKS_NODE); } else { - additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, clusterTemplate, networkIds, owner, + additionalControlVm = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, controlNodeTemplate, networkIds, owner, hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, - null, addrs, null, null, null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); + null, addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); } if (logger.isInfoEnabled()) { @@ -315,15 +462,62 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return additionalControlVm; } - private UserVm provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress) throws + private UserVm createEtcdNode(List requestedIps, List etcdNodeHostnames, int etcdNodeIndex, Long domainId, Long accountId) throws ResourceUnavailableException, InsufficientCapacityException, ResourceAllocationException { + UserVm etcdNode = null; + DataCenter zone = dataCenterDao.findById(kubernetesCluster.getZoneId()); + ServiceOffering serviceOffering = getServiceOfferingForNodeTypeOnCluster(ETCD, kubernetesCluster); + List networkIds = Collections.singletonList(kubernetesCluster.getNetworkId()); + Network.IpAddresses addrs = new Network.IpAddresses(null, null); + List guestIps = requestedIps.stream().map(Network.IpAddresses::getIp4Address).collect(Collectors.toList()); + String k8sControlNodeConfig = null; + try { + k8sControlNodeConfig = getEtcdNodeConfig(guestIps, etcdNodeHostnames, etcdNodeIndex, Hypervisor.HypervisorType.VMware.equals(clusterTemplate.getHypervisorType())); + } catch (IOException e) { + logAndThrow(Level.ERROR, "Failed to read Kubernetes control configuration file", e); + } + + String base64UserData = Base64.encodeBase64String(k8sControlNodeConfig.getBytes(com.cloud.utils.StringUtils.getPreferredCharset())); + List keypairs = new ArrayList(); + if (StringUtils.isNotBlank(kubernetesCluster.getKeyPair())) { + keypairs.add(kubernetesCluster.getKeyPair()); + } + Long affinityGroupId = getExplicitAffinityGroup(domainId, accountId); + String hostName = etcdNodeHostnames.get(etcdNodeIndex); + Map customParameterMap = new HashMap(); + if (zone.isSecurityGroupEnabled()) { + List securityGroupIds = new ArrayList<>(); + securityGroupIds.add(kubernetesCluster.getSecurityGroupId()); + etcdNode = userVmService.createAdvancedSecurityGroupVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, securityGroupIds, owner, + hostName, hostName, null, null, null, Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST,base64UserData, null, null, keypairs, + Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, + null, true, null, null); + } else { + etcdNode = userVmService.createAdvancedVirtualMachine(zone, serviceOffering, etcdTemplate, networkIds, owner, + hostName, hostName, null, null, null, + Hypervisor.HypervisorType.None, BaseCmd.HTTPMethod.POST, base64UserData, null, null, keypairs, + Map.of(kubernetesCluster.getNetworkId(), requestedIps.get(etcdNodeIndex)), addrs, null, null, Objects.nonNull(affinityGroupId) ? + Collections.singletonList(affinityGroupId) : null, customParameterMap, null, null, null, null, true, UserVmManager.CKS_NODE, null); + } + + if (logger.isInfoEnabled()) { + logger.info(String.format("Created control VM ID : %s, %s in the Kubernetes cluster : %s", etcdNode.getUuid(), hostName, kubernetesCluster.getName())); + } + return etcdNode; + } + + private Pair provisionKubernetesClusterControlVm(final Network network, final String publicIpAddress, final List etcdIps, + final Long domainId, final Long accountId, Long asNumber) throws ManagementServerException, InsufficientCapacityException, ResourceUnavailableException { UserVm k8sControlVM = null; - k8sControlVM = createKubernetesControlNode(network, publicIpAddress); - addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true); + Pair k8sControlVMAndControlIP; + k8sControlVMAndControlIP = createKubernetesControlNode(network, publicIpAddress, etcdIps, domainId, accountId, asNumber); + k8sControlVM = k8sControlVMAndControlIP.first(); + addKubernetesClusterVm(kubernetesCluster.getId(), k8sControlVM.getId(), true, false, false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(k8sControlVM); } - startKubernetesVM(k8sControlVM); + startKubernetesVM(k8sControlVM, domainId, accountId, CONTROL); k8sControlVM = userVmDao.findById(k8sControlVM.getId()); if (k8sControlVM == null) { throw new ManagementServerException(String.format("Failed to provision control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); @@ -331,21 +525,22 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif if (logger.isInfoEnabled()) { logger.info("Provisioned the control VM: {} in to the Kubernetes cluster: {}", k8sControlVM, kubernetesCluster); } - return k8sControlVM; + return new Pair<>(k8sControlVM, k8sControlVMAndControlIP.second()); } - private List provisionKubernetesClusterAdditionalControlVms(final String publicIpAddress) throws + private List provisionKubernetesClusterAdditionalControlVms(final String controlIpAddress, final Long domainId, + final Long accountId) throws InsufficientCapacityException, ManagementServerException, ResourceUnavailableException { List additionalControlVms = new ArrayList<>(); if (kubernetesCluster.getControlNodeCount() > 1) { for (int i = 1; i < kubernetesCluster.getControlNodeCount(); i++) { UserVm vm = null; - vm = createKubernetesAdditionalControlNode(publicIpAddress, i); - addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true); + vm = createKubernetesAdditionalControlNode(controlIpAddress, i, domainId, accountId); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), true, false, false, false); if (kubernetesCluster.getNodeRootDiskSize() > 0) { resizeNodeVolume(vm); } - startKubernetesVM(vm); + startKubernetesVM(vm, domainId, accountId, CONTROL); vm = userVmDao.findById(vm.getId()); if (vm == null) { throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); @@ -359,6 +554,35 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return additionalControlVms; } + private Pair, List> provisionEtcdCluster(final Network network, final Long domainId, final Long accountId) + throws InsufficientCapacityException, ResourceUnavailableException, ManagementServerException { + List etcdNodeVms = new ArrayList<>(); + List etcdNodeGuestIps = getEtcdNodeGuestIps(network, kubernetesCluster.getEtcdNodeCount()); + List etcdHostnames = getEtcdNodeHostnames(); + for (int i = 0; i < kubernetesCluster.getEtcdNodeCount(); i++) { + UserVm vm = createEtcdNode(etcdNodeGuestIps, etcdHostnames, i, domainId, accountId); + addKubernetesClusterVm(kubernetesCluster.getId(), vm.getId(), false, false, true, true); + startKubernetesVM(vm, domainId, accountId, ETCD); + vm = userVmDao.findById(vm.getId()); + if (vm == null) { + throw new ManagementServerException(String.format("Failed to provision additional control VM for Kubernetes cluster : %s" , kubernetesCluster.getName())); + } + etcdNodeVms.add(vm); + if (logger.isInfoEnabled()) { + logger.info(String.format("Provisioned additional control VM : %s in to the Kubernetes cluster : %s", vm.getDisplayName(), kubernetesCluster.getName())); + } + } + return new Pair<>(etcdNodeVms, etcdNodeGuestIps); + } + + private List getEtcdNodeGuestIps(final Network network, final long etcdNodeCount) { + List guestIps = new ArrayList<>(); + for (int i = 1; i <= etcdNodeCount; i++) { + guestIps.add(new Network.IpAddresses(ipAddressManager.acquireGuestIpAddress(network, null), null)); + } + return guestIps; + } + private Network startKubernetesClusterNetwork(final DeployDestination destination) throws ManagementServerException { final ReservationContext context = new ReservationContextImpl(null, null, null, owner); Network network = networkDao.findById(kubernetesCluster.getNetworkId()); @@ -406,7 +630,40 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif setupKubernetesClusterIsolatedNetworkRules(publicIp, network, clusterVMIds, true); } - private void startKubernetesClusterVMs() { + protected void setupKubernetesEtcdNetworkRules(List etcdVms, Network network) throws ManagementServerException, ResourceUnavailableException { + if (!Network.GuestType.Isolated.equals(network.getGuestType())) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("Network : %s for Kubernetes cluster : %s is not an isolated network, therefore, no need for network rules", network.getName(), kubernetesCluster.getName())); + } + } + List etcdVmIds = etcdVms.stream().map(UserVm::getId).collect(Collectors.toList()); + Integer startPort = KubernetesClusterService.KubernetesEtcdNodeStartPort.value(); + IpAddress publicIp = ipAddressDao.findByIpAndDcId(kubernetesCluster.getZoneId(), publicIpAddress); + for (int i = 0; i < etcdVmIds.size(); i++) { + int etcdStartPort = startPort + i; + try { + if (Objects.isNull(network.getVpcId())) { + provisionFirewallRules(publicIp, owner, etcdStartPort, etcdStartPort); + } else if (network.getNetworkACLId() != NetworkACL.DEFAULT_ALLOW) { + try { + provisionVpcTierAllowPortACLRule(network, ETCD_NODE_CLIENT_REQUEST_PORT, ETCD_NODE_CLIENT_REQUEST_PORT); + if (logger.isInfoEnabled()) { + logger.info(String.format("Provisioned ACL rule to open up port %d on %s for etcd nodes for Kubernetes cluster %s", + ETCD_NODE_CLIENT_REQUEST_PORT, publicIpAddress, kubernetesCluster.getName())); + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | InvalidParameterValueException | PermissionDeniedException e) { + throw new ManagementServerException(String.format("Failed to provision ACL rules for etcd client access for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + } + } catch (NoSuchFieldException | IllegalAccessException | ResourceUnavailableException | + NetworkRuleConflictException e) { + throw new ManagementServerException(String.format("Failed to provision firewall rules for etcd nodes for the Kubernetes cluster : %s", kubernetesCluster.getName()), e); + } + provisionPublicIpPortForwardingRule(publicIp, network, owner, etcdVmIds.get(i), etcdStartPort, DEFAULT_SSH_PORT); + } + } + + private void startKubernetesClusterVMs(Long domainId, Long accountId) { List clusterVms = getKubernetesClusterVMs(); for (final UserVm vm : clusterVms) { if (vm == null) { @@ -414,7 +671,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } try { resizeNodeVolume(vm); - startKubernetesVM(vm); + KubernetesClusterVmMapVO map = kubernetesClusterVmMapDao.findByVmId(vm.getId()); + KubernetesServiceHelper.KubernetesClusterNodeType nodeType = getNodeTypeFromClusterVMMapRecord(map); + startKubernetesVM(vm, domainId, accountId, nodeType); } catch (ManagementServerException ex) { logger.warn("Failed to start VM: {} in Kubernetes cluster: {} due to {}", vm, kubernetesCluster, ex); // don't bail out here. proceed further to stop the reset of the VM's @@ -428,6 +687,16 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } } + private KubernetesServiceHelper.KubernetesClusterNodeType getNodeTypeFromClusterVMMapRecord(KubernetesClusterVmMapVO map) { + if (map.isControlNode()) { + return CONTROL; + } else if (map.isEtcdNode()) { + return ETCD; + } else { + return WORKER; + } + } + private boolean isKubernetesClusterKubeConfigAvailable(final long timeoutTime) { if (StringUtils.isEmpty(publicIpAddress)) { KubernetesClusterDetailsVO kubeConfigDetail = kubernetesClusterDetailsDao.findDetail(kubernetesCluster.getId(), "kubeConfigData"); @@ -468,7 +737,7 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif kubernetesClusterDao.update(kubernetesCluster.getId(), kubernetesClusterVO); } - public boolean startKubernetesClusterOnCreate() { + public boolean startKubernetesClusterOnCreate(Long domainId, Long accountId, Long asNumber) throws ManagementServerException, ResourceUnavailableException, InsufficientCapacityException { init(); if (logger.isInfoEnabled()) { logger.info("Starting Kubernetes cluster: {}", kubernetesCluster); @@ -477,7 +746,9 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); DeployDestination dest = null; try { - dest = plan(); + VMTemplateVO clusterTemplate = templateDao.findById(kubernetesCluster.getTemplateId()); + Map destinationMap = planKubernetesCluster(domainId, accountId, clusterTemplate.getHypervisorType()); + dest = destinationMap.get(WORKER.name()); } catch (InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the cluster failed due to insufficient capacity in the Kubernetes cluster: %s", kubernetesCluster.getUuid()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } @@ -499,16 +770,28 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif logTransitStateAndThrow(Level.ERROR, String.format("Failed to start Kubernetes cluster : %s as no public IP found for the cluster" , kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed); } // Allow account creating the kubernetes cluster to access systemVM template - LaunchPermissionVO launchPermission = new LaunchPermissionVO(clusterTemplate.getId(), owner.getId()); - launchPermissionDao.persist(launchPermission); + if (isDefaultTemplateUsed()) { + LaunchPermissionVO launchPermission = new LaunchPermissionVO(kubernetesCluster.getTemplateId(), owner.getId()); + launchPermissionDao.persist(launchPermission); + } + + List etcdVms = new ArrayList<>(); + List etcdGuestNodeIps = new ArrayList<>(); + if (kubernetesCluster.getEtcdNodeCount() > 0) { + Pair, List> etcdNodesAndIps = provisionEtcdCluster(network, domainId, accountId); + etcdVms = etcdNodesAndIps.first(); + etcdGuestNodeIps = etcdNodesAndIps.second(); + } List clusterVMs = new ArrayList<>(); + Pair k8sControlVMAndIp = new Pair<>(null, null); UserVm k8sControlVM = null; try { - k8sControlVM = provisionKubernetesClusterControlVm(network, publicIpAddress); + k8sControlVMAndIp = provisionKubernetesClusterControlVm(network, publicIpAddress, etcdGuestNodeIps, domainId, accountId, asNumber); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning the control VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } + k8sControlVM = k8sControlVMAndIp.first(); clusterVMs.add(k8sControlVM); if (StringUtils.isEmpty(publicIpAddress)) { publicIpSshPort = getKubernetesClusterServerIpSshPort(k8sControlVM); @@ -518,13 +801,13 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } } try { - List additionalControlVMs = provisionKubernetesClusterAdditionalControlVms(publicIpAddress); + List additionalControlVMs = provisionKubernetesClusterAdditionalControlVms(k8sControlVMAndIp.second(), domainId, accountId); clusterVMs.addAll(additionalControlVMs); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning additional control VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } try { - List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), publicIpAddress); + List nodeVMs = provisionKubernetesClusterNodeVms(kubernetesCluster.getNodeCount(), k8sControlVMAndIp.second(), domainId, accountId); clusterVMs.addAll(nodeVMs); } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { logTransitStateAndThrow(Level.ERROR, String.format("Provisioning node VM failed in the Kubernetes cluster : %s", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); @@ -537,6 +820,12 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif } catch (ManagementServerException e) { logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s, unable to setup network rules", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); } + try { + setupKubernetesEtcdNetworkRules(etcdVms, network); + } catch (ManagementServerException e) { + logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s, unable to setup network rules for etcd nodes", kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.CreateFailed, e); + } + attachIsoKubernetesVMs(etcdVms); attachIsoKubernetesVMs(clusterVMs); if (!KubernetesClusterUtil.isKubernetesClusterControlVmRunning(kubernetesCluster, publicIpAddress, publicIpSshPort.second(), startTimeoutTime)) { String msg = String.format("Failed to setup Kubernetes cluster : %s is not in usable state as the system is unable to access control node VMs of the cluster", kubernetesCluster.getName()); @@ -574,14 +863,16 @@ public class KubernetesClusterStartWorker extends KubernetesClusterResourceModif return true; } - public boolean startStoppedKubernetesCluster() throws CloudRuntimeException { + + + public boolean startStoppedKubernetesCluster(Long domainId, Long accountId) throws CloudRuntimeException { init(); if (logger.isInfoEnabled()) { logger.info("Starting Kubernetes cluster: {}", kubernetesCluster); } final long startTimeoutTime = System.currentTimeMillis() + KubernetesClusterService.KubernetesClusterStartTimeout.value() * 1000; stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.StartRequested); - startKubernetesClusterVMs(); + startKubernetesClusterVMs(domainId, accountId); try { InetAddress address = InetAddress.getByName(new URL(kubernetesCluster.getEndpoint()).getHost()); } catch (MalformedURLException | UnknownHostException ex) { diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index ab3121f207b..4c2725fc2a2 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -20,7 +20,10 @@ package com.cloud.kubernetes.cluster.actionworkers; import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; @@ -40,7 +43,7 @@ import com.cloud.utils.ssh.SshHelper; public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorker { - private List clusterVMs = new ArrayList<>(); + protected List clusterVMs = new ArrayList<>(); private KubernetesSupportedVersion upgradeVersion; private final String upgradeScriptFilename = "upgrade-kubernetes.sh"; private File upgradeScriptFile; @@ -65,12 +68,12 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke String nodeAddress = (index > 0 && sshPort == 22) ? vm.getPrivateIpAddress() : publicIpAddress; SshHelper.scpTo(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, "~/", upgradeScriptFile.getAbsolutePath(), "0755"); - String cmdStr = String.format("sudo ./%s %s %s %s %s", + String cmdStr = String.format("sudo ./%s %s %s %s %s %s", upgradeScriptFile.getName(), upgradeVersion.getSemanticVersion(), index == 0 ? "true" : "false", KubernetesVersionManagerImpl.compareSemanticVersions(upgradeVersion.getSemanticVersion(), "1.15.0") < 0 ? "true" : "false", - Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType())); + Hypervisor.HypervisorType.VMware.equals(vm.getHypervisorType()), Objects.isNull(kubernetesCluster.getCniConfigId())); return SshHelper.sshExecute(nodeAddress, nodeSshPort, getControlNodeLoginUser(), sshKeyFile, null, cmdStr, 10000, 10000, 10 * 60 * 1000); @@ -144,7 +147,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get control Kubernetes node on VM : %s in ready state", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } } - if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000)) { + if (!KubernetesClusterUtil.clusterNodeVersionMatches(upgradeVersion.getSemanticVersion(), publicIpAddress, sshPort, getControlNodeLoginUser(), getManagementServerSshPublicKeyFile(), hostName, upgradeTimeoutTime, 15000, vm.getId(), kubernetesClusterVmMapDao)) { logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to get Kubernetes node on VM : %s upgraded to version %s", kubernetesCluster.getName(), vm.getDisplayName(), upgradeVersion.getSemanticVersion()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null); } if (logger.isInfoEnabled()) { @@ -169,6 +172,7 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke if (CollectionUtils.isEmpty(clusterVMs)) { logAndThrow(Level.ERROR, String.format("Upgrade failed for Kubernetes cluster: %s, unable to retrieve VMs for cluster", kubernetesCluster)); } + filterOutManualUpgradeNodesFromClusterUpgrade(); retrieveScriptFiles(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.UpgradeRequested); attachIsoKubernetesVMs(clusterVMs, upgradeVersion); @@ -184,4 +188,14 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke } return updated; } + + protected void filterOutManualUpgradeNodesFromClusterUpgrade() { + if (CollectionUtils.isEmpty(clusterVMs)) { + return; + } + clusterVMs = clusterVMs.stream().filter(x -> { + KubernetesClusterVmMapVO mapVO = kubernetesClusterVmMapDao.getClusterMapFromVmId(x.getId()); + return mapVO != null && !mapVO.isManualUpgrade(); + }).collect(Collectors.toList()); + } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java index eaeccd09f80..71b90ef2e6a 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.kubernetes.cluster.dao; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; import com.cloud.utils.db.GenericDao; @@ -31,5 +32,7 @@ public interface KubernetesClusterVmMapDao extends GenericDao listByClusterIdAndVmType(long clusterId, KubernetesClusterNodeType nodeType); + KubernetesClusterVmMapVO findByVmId(long vmId); } diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java index 5e465848e1c..9607783b50e 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java @@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster.dao; import java.util.List; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; import org.springframework.stereotype.Component; import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; @@ -26,6 +27,9 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; + @Component public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase implements KubernetesClusterVmMapDao { @@ -37,6 +41,8 @@ public class KubernetesClusterVmMapDaoImpl extends GenericDaoBase listByClusterIdAndVmType(long clusterId, KubernetesServiceHelper.KubernetesClusterNodeType nodeType) { + SearchCriteria sc = clusterIdSearch.create(); + sc.setParameters("clusterId", clusterId); + if (CONTROL == nodeType) { + sc.setParameters("controlNode", true); + sc.setParameters("etcdNode", false); + } else if (ETCD == nodeType) { + sc.setParameters("controlNode", false); + sc.setParameters("etcdNode", true); + } else { + sc.setParameters("controlNode", false); + sc.setParameters("etcdNode", false); + } + return listBy(sc); + } + @Override public KubernetesClusterVmMapVO findByVmId(long vmId) { SearchBuilder sb = createSearchBuilder(); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java index 0195a5a7916..00625f6e076 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtil.java @@ -31,6 +31,8 @@ import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import org.apache.cloudstack.utils.security.SSLUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; @@ -215,10 +217,10 @@ public class KubernetesClusterUtil { final int port, final String user, final File sshKeyFile) throws Exception { Pair result = SshHelper.sshExecute(ipAddress, port, user, sshKeyFile, null, - "sudo /opt/bin/kubectl get nodes | awk '{if ($2 == \"Ready\") print $1}' | wc -l", + "sudo /opt/bin/kubectl get nodes | grep -w 'Ready' | wc -l", 10000, 10000, 20000); - if (result.first()) { - return Integer.parseInt(result.second().trim().replace("\"", "")); + if (Boolean.TRUE.equals(result.first())) { + return Integer.parseInt(result.second().trim().replace("\"", "")) + kubernetesCluster.getEtcdNodeCount().intValue(); } else { if (LOGGER.isDebugEnabled()) { LOGGER.debug(String.format("Failed to retrieve ready nodes for Kubernetes cluster %s. Output: %s", kubernetesCluster, result.second())); @@ -331,7 +333,7 @@ public class KubernetesClusterUtil { final String ipAddress, final int port, final String user, final File sshKeyFile, final String hostName, - final long timeoutTime, final long waitDuration) { + final long timeoutTime, final long waitDuration, final long vmId, KubernetesClusterVmMapDao vmMapDao) { int retry = 10; while (System.currentTimeMillis() < timeoutTime && retry-- > 0) { if (LOGGER.isDebugEnabled()) { @@ -343,7 +345,13 @@ public class KubernetesClusterUtil { user, sshKeyFile, null, String.format(CLUSTER_NODE_VERSION_COMMAND, hostName.toLowerCase()), 10000, 10000, 20000); - if (clusterNodeVersionMatches(result, version)) { + Pair clusterVersionMatchesAndValue = clusterNodeVersionMatches(result, version); + if (Boolean.TRUE.equals(clusterVersionMatchesAndValue.first())) { + KubernetesClusterVmMapVO vmMapVO = vmMapDao.getClusterMapFromVmId(vmId); + String newNodeVersion = clusterVersionMatchesAndValue.second(); + LOGGER.debug(String.format("Updating node %s Kubernetes version to %s", hostName, newNodeVersion)); + vmMapVO.setNodeVersion(newNodeVersion); + vmMapDao.update(vmMapVO.getId(), vmMapVO); return true; } } catch (Exception e) { @@ -360,11 +368,11 @@ public class KubernetesClusterUtil { return false; } - protected static boolean clusterNodeVersionMatches(final Pair result, final String version) { + protected static Pair clusterNodeVersionMatches(final Pair result, final String version) { if (result == null || Boolean.FALSE.equals(result.first()) || StringUtils.isBlank(result.second())) { - return false; + return new Pair<>(false, null); } String response = result.second(); - return response.contains(String.format("v%s", version)); + return new Pair<>(response.contains(String.format("v%s", version)), response); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java new file mode 100644 index 00000000000..71cc8ffcde2 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/AddNodesToKubernetesClusterCmd.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.lang3.BooleanUtils; + +import javax.inject.Inject; + +import java.util.List; + +@APICommand(name = "addNodesToKubernetesCluster", + description = "Add nodes as workers to an existing CKS cluster. ", + responseObject = KubernetesClusterResponse.class, + since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class AddNodesToKubernetesClusterCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + @Parameter(name = ApiConstants.NODE_IDS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType= UserVmResponse.class, + description = "comma separated list of (external) node (physical or virtual machines) IDs that need to be" + + "added as worker nodes to an existing managed Kubernetes cluster (CKS)", + required = true) + private List nodeIds; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster", since = "4.21.0") + private Long clusterId; + + @Parameter(name = ApiConstants.MOUNT_CKS_ISO_ON_VR, type = CommandType.BOOLEAN, + description = "(optional) Vmware only, uses the CKS cluster network VR to mount the CKS ISO") + private Boolean mountCksIsoOnVr; + + @Parameter(name = ApiConstants.MANUAL_UPGRADE, type = CommandType.BOOLEAN, + description = "(optional) indicates if the node is marked for manual upgrade and excluded from the Kubernetes cluster upgrade operation") + private Boolean manualUpgrade; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public List getNodeIds() { + return nodeIds; + } + + public Long getClusterId() { + return clusterId; + } + + public boolean isMountCksIsoOnVr() { + return BooleanUtils.isTrue(mountCksIsoOnVr); + } + + public boolean isManualUpgrade() { + return BooleanUtils.isTrue(manualUpgrade); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_ADD; + } + + @Override + public String getEventDescription() { + return String.format("Adding %s nodes to the Kubernetes cluster with ID: %s", nodeIds.size(), clusterId); + } + + @Override + public void execute() { + try { + kubernetesClusterService.addNodesToKubernetesCluster(this); + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getClusterId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to add nodes to cluster ID %s due to: %s", + getClusterId(), e.getLocalizedMessage()), e); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getClusterId(); + } + +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java index 721cb47867b..4e92f9546f8 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/CreateKubernetesClusterCmd.java @@ -16,9 +16,26 @@ // under the License. package org.apache.cloudstack.api.command.user.kubernetes.cluster; +import java.security.InvalidParameterException; +import java.util.Map; +import java.util.Objects; + import javax.inject.Inject; +import com.cloud.dc.ASNumberVO; +import com.cloud.dc.dao.ASNumberDao; +import com.cloud.exception.InsufficientCapacityException; import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offering.NetworkOffering; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.utils.Pair; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; @@ -36,8 +53,11 @@ import org.apache.cloudstack.api.response.KubernetesSupportedVersionResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.commons.lang3.StringUtils; import com.cloud.kubernetes.cluster.KubernetesCluster; @@ -58,6 +78,16 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Inject public KubernetesClusterService kubernetesClusterService; + @Inject + protected KubernetesServiceHelper kubernetesClusterHelper; + @Inject + private ConfigurationDao configurationDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOfferingDao networkOfferingDao; + @Inject + private ASNumberDao asNumberDao; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -83,6 +113,25 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { description = "the ID of the service offering for the virtual machines in the cluster.") private Long serviceOfferingId; + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Service Offering ID mapping. If provided, it overrides the serviceofferingid parameter", + since = "4.21.0") + private Map> serviceOfferingNodeTypeMap; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_TEMPLATE_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Template ID mapping. If provided, it overrides the default template: System VM template", + since = "4.21.0") + private Map> templateNodeTypeMap; + + @ACL(accessType = AccessType.UseEntry) + @Parameter(name = ApiConstants.ETCD_NODES, type = CommandType.LONG, + description = "(Optional) Number of Kubernetes cluster etcd nodes, default is 0." + + "In case the number is greater than 0, etcd nodes are separate from master nodes and are provisioned accordingly", + since = "4.21.0") + private Long etcdNodes; + @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "an optional account for the" + " virtual machine. Must be used with domainId.") @@ -90,7 +139,8 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @ACL(accessType = AccessType.UseEntry) @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, - description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used.") + description = "an optional domainId for the virtual machine. If the account parameter is used, domainId must also be used. " + + "Hosts dedicated to the specified domain will be used for deploying the cluster") private Long domainId; @ACL(accessType = AccessType.UseEntry) @@ -144,6 +194,22 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.CLUSTER_TYPE, type = CommandType.STRING, description = "type of the cluster: CloudManaged, ExternalManaged. The default value is CloudManaged.", since="4.19.0") private String clusterType; + @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "the hypervisor on which the CKS cluster is to be deployed. This is required if the zone in which the CKS cluster is being deployed has clusters with different hypervisor types.", since = "4.21.0") + private String hypervisor; + + @Parameter(name = ApiConstants.CNI_CONFIG_ID, type = CommandType.UUID, entityType = UserDataResponse.class, description = "the ID of the Userdata", since = "4.21.0") + private Long cniConfigId; + + @Parameter(name = ApiConstants.CNI_CONFIG_DETAILS, type = CommandType.MAP, + description = "used to specify the parameters values for the variables in userdata. " + + "Example: cniconfigdetails[0].key=accesskey&cniconfigdetails[0].value=s389ddssaa&" + + "cniconfigdetails[1].key=secretkey&cniconfigdetails[1].value=8dshfsss", + since = "4.21.0") + private Map cniConfigDetails; + + @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, description="the AS Number of the network") + private Long asNumber; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -202,6 +268,10 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return controlNodes; } + public long getEtcdNodes() { + return etcdNodes == null ? 0 : etcdNodes; + } + public String getExternalLoadBalancerIpAddress() { return externalLoadBalancerIpAddress; } @@ -240,6 +310,67 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { return clusterType; } + public Map getServiceOfferingNodeTypeMap() { + return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + } + + public Map getTemplateNodeTypeMap() { + return kubernetesClusterHelper.getTemplateNodeTypeMap(templateNodeTypeMap); + } + + public Hypervisor.HypervisorType getHypervisorType() { + return hypervisor == null ? null : Hypervisor.HypervisorType.getType(hypervisor); + } + + private Pair getKubernetesNetworkOffering(Long networkId) { + if (Objects.isNull(networkId)) { + ConfigurationVO configurationVO = configurationDao.findByName(KubernetesClusterService.KubernetesClusterNetworkOffering.key()); + String offeringName = configurationVO.getValue(); + return new Pair<>(networkOfferingDao.findByUniqueName(offeringName), null); + } else { + NetworkVO networkVO = networkDao.findById(getNetworkId()); + if (networkVO == null) { + throw new InvalidParameterException(String.format("Failed to find network with id: %s", getNetworkId())); + } + NetworkOfferingVO offeringVO = networkOfferingDao.findById(networkVO.getNetworkOfferingId()); + return new Pair<>(offeringVO, networkVO); + } + } + + public Long getAsNumber() { + Pair offeringAndNetwork = getKubernetesNetworkOffering(getNetworkId()); + NetworkOfferingVO offering = offeringAndNetwork.first(); + NetworkVO networkVO = offeringAndNetwork.second(); + + if (offering == null) { + throw new CloudRuntimeException("Failed to find kubernetes network offering"); + } + ASNumberVO asNumberVO = null; + if (Objects.isNull(getNetworkId()) && !offering.isForVpc()) { + if (Boolean.TRUE.equals(NetworkOffering.RoutingMode.Dynamic.equals(offering.getRoutingMode()) && offering.isSpecifyAsNumber()) && asNumber == null) { + throw new InvalidParameterException("AsNumber must be specified as network offering has specifyasnumber set"); + } + } else if (Objects.nonNull(networkVO)) { + if (offering.isForVpc()) { + asNumberVO = asNumberDao.findByZoneAndVpcId(getZoneId(), networkVO.getVpcId()); + } else { + asNumberVO = asNumberDao.findByZoneAndNetworkId(getZoneId(), getNetworkId()); + } + } + if (Objects.nonNull(asNumberVO)) { + return asNumberVO.getAsNumber(); + } + return asNumber; + } + + public Map getCniConfigDetails() { + return convertDetailsToMap(cniConfigDetails); + } + + public Long getCniConfigId() { + return cniConfigId; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -290,7 +421,7 @@ public class CreateKubernetesClusterCmd extends BaseAsyncCreateCmd { KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getEntityId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (CloudRuntimeException e) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | InsufficientCapacityException e) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, e.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java new file mode 100644 index 00000000000..2ca36e83ab4 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/RemoveNodesFromKubernetesClusterCmd.java @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.kubernetes.cluster; + +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.kubernetes.cluster.KubernetesClusterEventTypes; +import com.cloud.kubernetes.cluster.KubernetesClusterService; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.KubernetesClusterResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import javax.inject.Inject; +import java.util.List; + +@APICommand(name = "removeNodesFromKubernetesCluster", + description = "Removes external nodes from a CKS cluster. ", + responseObject = KubernetesClusterResponse.class, + since = "4.21.0", + authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User}) +public class RemoveNodesFromKubernetesClusterCmd extends BaseAsyncCmd { + + @Inject + public KubernetesClusterService kubernetesClusterService; + + protected static final Logger LOGGER = LogManager.getLogger(RemoveNodesFromKubernetesClusterCmd.class); + + @Parameter(name = ApiConstants.NODE_IDS, + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType= UserVmResponse.class, + description = "comma separated list of node (physical or virtual machines) IDs that need to be" + + "removed from the Kubernetes cluster (CKS)", + required = true) + private List nodeIds; + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = KubernetesClusterResponse.class, + description = "the ID of the Kubernetes cluster") + private Long clusterId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public List getNodeIds() { + return nodeIds; + } + + public Long getClusterId() { + return clusterId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getEventType() { + return KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_NODES_REMOVE; + } + + @Override + public String getEventDescription() { + return String.format("Removing %s nodes from the Kubernetes Cluster with ID: %s", nodeIds.size(), clusterId); + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + try { + if (!kubernetesClusterService.removeNodesFromKubernetesCluster(this)) { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, String.format("Failed to remove node(s) from Kubernetes cluster ID: %d", getClusterId())); + } + final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getClusterId()); + response.setResponseName(getCommandName()); + setResponseObject(response); + } catch (Exception e) { + String err = String.format("Failed to remove node(s) from Kubernetes cluster ID: %d due to: %s", getClusterId(), e.getMessage()); + LOGGER.error(err, e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, err); + } + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.KubernetesCluster; + } + + @Override + public Long getApiResourceId() { + return getClusterId(); + } +} diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java index 59c2bebf961..2fb92f60fb9 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/ScaleKubernetesClusterCmd.java @@ -17,9 +17,11 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import java.util.List; +import java.util.Map; import javax.inject.Inject; +import com.cloud.kubernetes.cluster.KubernetesServiceHelper; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.api.ACL; @@ -54,6 +56,8 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { @Inject public KubernetesClusterService kubernetesClusterService; + @Inject + protected KubernetesServiceHelper kubernetesClusterHelper; ///////////////////////////////////////////////////// //////////////// API parameters ///////////////////// @@ -68,6 +72,12 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { description = "the ID of the service offering for the virtual machines in the cluster.") private Long serviceOfferingId; + @ACL(accessType = SecurityChecker.AccessType.UseEntry) + @Parameter(name = ApiConstants.NODE_TYPE_OFFERING_MAP, type = CommandType.MAP, + description = "(Optional) Node Type to Service Offering ID mapping. If provided, it overrides the serviceofferingid parameter", + since = "4.21.0") + protected Map> serviceOfferingNodeTypeMap; + @Parameter(name=ApiConstants.SIZE, type = CommandType.LONG, description = "number of Kubernetes cluster nodes") private Long clusterSize; @@ -103,6 +113,10 @@ public class ScaleKubernetesClusterCmd extends BaseAsyncCmd { return serviceOfferingId; } + public Map getServiceOfferingNodeTypeMap() { + return kubernetesClusterHelper.getServiceOfferingNodeTypeMap(this.serviceOfferingNodeTypeMap); + } + public Long getClusterSize() { return clusterSize; } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java index bfe00ca27b2..50e8202b8b0 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/command/user/kubernetes/cluster/StartKubernetesClusterCmd.java @@ -18,6 +18,9 @@ package org.apache.cloudstack.api.command.user.kubernetes.cluster; import javax.inject.Inject; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ManagementServerException; +import com.cloud.exception.ResourceUnavailableException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; @@ -102,7 +105,8 @@ public class StartKubernetesClusterCmd extends BaseAsyncCmd { final KubernetesClusterResponse response = kubernetesClusterService.createKubernetesClusterResponse(getId()); response.setResponseName(getCommandName()); setResponseObject(response); - } catch (CloudRuntimeException ex) { + } catch (CloudRuntimeException | ManagementServerException | ResourceUnavailableException | + InsufficientCapacityException ex) { throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, ex.getMessage()); } } diff --git a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java index 8074aef9eff..b811f4f9dcb 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java +++ b/plugins/integrations/kubernetes-service/src/main/java/org/apache/cloudstack/api/response/KubernetesClusterResponse.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.response; import java.util.Date; import java.util.List; +import java.util.Map; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithAnnotations; @@ -58,6 +59,34 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "the name of the service offering of the Kubernetes cluster") private String serviceOfferingName; + @SerializedName(ApiConstants.WORKER_SERVICE_OFFERING_ID) + @Param(description = "the ID of the service offering of the worker nodes on the Kubernetes cluster") + private String workerOfferingId; + + @SerializedName(ApiConstants.WORKER_SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering of the worker nodes on the Kubernetes cluster") + private String workerOfferingName; + + @SerializedName(ApiConstants.CONTROL_SERVICE_OFFERING_ID) + @Param(description = "the ID of the service offering of the control nodes on the Kubernetes cluster") + private String controlOfferingId; + + @SerializedName(ApiConstants.CONTROL_SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering of the control nodes on the Kubernetes cluster") + private String controlOfferingName; + + @SerializedName(ApiConstants.ETCD_SERVICE_OFFERING_ID) + @Param(description = "the ID of the service offering of the etcd nodes on the Kubernetes cluster") + private String etcdOfferingId; + + @SerializedName(ApiConstants.ETCD_SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering of the etcd nodes on the Kubernetes cluster") + private String etcdOfferingName; + + @SerializedName(ApiConstants.ETCD_NODES) + @Param(description = "the number of the etcd nodes on the Kubernetes cluster") + private Long etcdNodes; + @SerializedName(ApiConstants.TEMPLATE_ID) @Param(description = "the ID of the template of the Kubernetes cluster") private String templateId; @@ -106,6 +135,14 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "keypair details") private String keypair; + @SerializedName(ApiConstants.CNI_CONFIG_ID) + @Param(description = "ID of CNI Configuration associated with the cluster") + private String cniConfigId; + + @SerializedName(ApiConstants.CNI_CONFIG_NAME) + @Param(description = "Name of CNI Configuration associated with the cluster") + private String cniConfigName; + @Deprecated(since = "4.16") @SerializedName(ApiConstants.MASTER_NODES) @Param(description = "the master nodes count for the Kubernetes cluster. This parameter is deprecated, please use 'controlnodes' parameter.") @@ -141,7 +178,7 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @SerializedName(ApiConstants.VIRTUAL_MACHINES) @Param(description = "the list of virtualmachine associated with this Kubernetes cluster") - private List virtualMachines; + private List virtualMachines; @SerializedName(ApiConstants.IP_ADDRESS) @Param(description = "Public IP Address of the cluster") @@ -151,6 +188,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple @Param(description = "Public IP Address ID of the cluster") private String ipAddressId; + @SerializedName(ApiConstants.ETCD_IPS) + @Param(description = "Public IP Addresses of the etcd nodes") + private Map etcdIps; + @SerializedName(ApiConstants.AUTOSCALING_ENABLED) @Param(description = "Whether autoscaling is enabled for the cluster") private boolean isAutoscalingEnabled; @@ -367,11 +408,67 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple this.serviceOfferingName = serviceOfferingName; } - public void setVirtualMachines(List virtualMachines) { + public String getWorkerOfferingId() { + return workerOfferingId; + } + + public void setWorkerOfferingId(String workerOfferingId) { + this.workerOfferingId = workerOfferingId; + } + + public String getWorkerOfferingName() { + return workerOfferingName; + } + + public void setWorkerOfferingName(String workerOfferingName) { + this.workerOfferingName = workerOfferingName; + } + + public String getControlOfferingId() { + return controlOfferingId; + } + + public void setControlOfferingId(String controlOfferingId) { + this.controlOfferingId = controlOfferingId; + } + + public String getControlOfferingName() { + return controlOfferingName; + } + + public void setControlOfferingName(String controlOfferingName) { + this.controlOfferingName = controlOfferingName; + } + + public String getEtcdOfferingId() { + return etcdOfferingId; + } + + public void setEtcdOfferingId(String etcdOfferingId) { + this.etcdOfferingId = etcdOfferingId; + } + + public String getEtcdOfferingName() { + return etcdOfferingName; + } + + public void setEtcdOfferingName(String etcdOfferingName) { + this.etcdOfferingName = etcdOfferingName; + } + + public Long getEtcdNodes() { + return etcdNodes; + } + + public void setEtcdNodes(Long etcdNodes) { + this.etcdNodes = etcdNodes; + } + + public void setVirtualMachines(List virtualMachines) { this.virtualMachines = virtualMachines; } - public List getVirtualMachines() { + public List getVirtualMachines() { return virtualMachines; } @@ -383,6 +480,10 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple this.ipAddressId = ipAddressId; } + public void setEtcdIps(Map etcdIps) { + this.etcdIps = etcdIps; + } + public void setAutoscalingEnabled(boolean isAutoscalingEnabled) { this.isAutoscalingEnabled = isAutoscalingEnabled; } @@ -406,4 +507,12 @@ public class KubernetesClusterResponse extends BaseResponseWithAnnotations imple public void setClusterType(KubernetesCluster.ClusterType clusterType) { this.clusterType = clusterType; } + + public void setCniConfigId(String cniConfigId) { + this.cniConfigId = cniConfigId; + } + + public void setCniConfigName(String cniConfigName) { + this.cniConfigName = cniConfigName; + } } diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml new file mode 100644 index 00000000000..d455441279d --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml @@ -0,0 +1,134 @@ +#cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +users: + - name: cloud + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + {{ k8s.ssh.pub.key }} + +write_files: + - path: /opt/bin/setup-etcd-node + permissions: '0700' + owner: root:root + content: | + #!/bin/bash -e + + if [[ -f "/home/cloud/success" ]]; then + echo "Already provisioned!" + exit 0 + fi + + ISO_MOUNT_DIR=/mnt/etcddisk + BINARIES_DIR=${ISO_MOUNT_DIR}/ + ATTEMPT_ONLINE_INSTALL=false + setup_complete=false + + OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }} + MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }} + if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + fi + if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then + MAX_OFFLINE_INSTALL_ATTEMPTS=100 + fi + offline_attempts=1 + MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 + EJECT_ISO_FROM_OS={{ k8s.eject.iso }} + crucial_cmd_attempts=1 + iso_drive_path="" + while true; do + if (( "$offline_attempts" > "$MAX_OFFLINE_INSTALL_ATTEMPTS" )); then + echo "Warning: Offline install timed out!" + break + fi + set +e + output=`blkid -o device -t TYPE=iso9660` + set -e + if [ "$output" != "" ]; then + while read -r line; do + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + retval=0 + set +e + mount -o ro "${line}" "${ISO_MOUNT_DIR}" + retval=$? + set -e + if [ $retval -eq 0 ]; then + if [ -d "$BINARIES_DIR" ]; then + iso_drive_path="${line}" + break + else + umount "${line}" && rmdir "${ISO_MOUNT_DIR}" + fi + fi + done <<< "$output" + fi + if [ -d "$BINARIES_DIR" ]; then + break + fi + echo "Waiting for Binaries directory $BINARIES_DIR to be available, sleeping for $OFFLINE_INSTALL_ATTEMPT_SLEEP seconds, attempt: $offline_attempts" + sleep $OFFLINE_INSTALL_ATTEMPT_SLEEP + offline_attempts=$[$offline_attempts + 1] + done + + if [[ "$PATH" != *:/opt/bin && "$PATH" != *:/opt/bin:* ]]; then + export PATH=$PATH:/opt/bin + fi + + if [ -d "$BINARIES_DIR" ]; then + ### Binaries available offline ### + echo "Installing binaries from ${BINARIES_DIR}" + mkdir -p /opt/bin/ + tar -zxf ${BINARIES_DIR}/etcd/etcd-linux-amd64.tar.gz -C /opt/bin/ + mv /opt/bin/etcd*/etcd* /opt/bin/ + sudo rm -rf /opt/bin/etcd-* + fi + + - path: /etc/systemd/system/etcd.service + permissions: '0755' + owner: root:root + content: | + [Unit] + Description=etcd + + [Service] + Type=exec + ExecStart=/opt/bin/etcd \ + --name {{ etcd.node_name }} \ + --initial-advertise-peer-urls http://{{ etcd.node_ip }}:2380 \ + --listen-peer-urls http://{{ etcd.node_ip }}:2380 \ + --advertise-client-urls http://{{ etcd.node_ip }}:2379 \ + --listen-client-urls http://{{ etcd.node_ip }}:2379,http://127.0.0.1:2379 \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster {{ etcd.initial_cluster_nodes }} \ + --initial-cluster-state new + Restart=on-failure + RestartSec=5 + + [Install] + WantedBy=multi-user.target + +runcmd: + - chown -R cloud:cloud /home/cloud/.ssh + - /opt/bin/setup-etcd-node + - systemctl daemon-reload + - systemctl enable --now etcd diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml index 6819723b3f0..38f217f403c 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml @@ -40,8 +40,8 @@ write_files: sysctl net.ipv4.conf.default.arp_ignore=0 sysctl net.ipv4.conf.all.arp_announce=0 sysctl net.ipv4.conf.all.arp_ignore=0 - sysctl net.ipv4.conf.eth0.arp_announce=0 - sysctl net.ipv4.conf.eth0.arp_ignore=0 + sysctl net.ipv4.conf.eth0.arp_announce=0 || sysctl net.ipv4.conf.ens35.arp_announce=0 || true + sysctl net.ipv4.conf.eth0.arp_ignore=0 || sysctl net.ipv4.conf.ens35.arp_ignore=0 || true sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf @@ -53,8 +53,14 @@ write_files: ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=15 - MAX_OFFLINE_INSTALL_ATTEMPTS=100 + OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }} + MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }} + if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + fi + if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then + MAX_OFFLINE_INSTALL_ATTEMPTS=100 + fi offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 EJECT_ISO_FROM_OS={{ k8s.eject.iso }} diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml index 90be8957d44..dc066e10d06 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml @@ -1,3 +1,4 @@ +## template: jinja #cloud-config # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -60,8 +61,8 @@ write_files: sysctl net.ipv4.conf.default.arp_ignore=0 sysctl net.ipv4.conf.all.arp_announce=0 sysctl net.ipv4.conf.all.arp_ignore=0 - sysctl net.ipv4.conf.eth0.arp_announce=0 - sysctl net.ipv4.conf.eth0.arp_ignore=0 + sysctl net.ipv4.conf.eth0.arp_announce=0 || sysctl net.ipv4.conf.ens35.arp_announce=0 || true + sysctl net.ipv4.conf.eth0.arp_ignore=0 || sysctl net.ipv4.conf.ens35.arp_ignore=0 || true sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf @@ -73,8 +74,14 @@ write_files: ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=15 - MAX_OFFLINE_INSTALL_ATTEMPTS=100 + OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }} + MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }} + if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then + OFFLINE_INSTALL_ATTEMPT_SLEEP=15 + fi + if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then + MAX_OFFLINE_INSTALL_ATTEMPTS=100 + fi offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 EJECT_ISO_FROM_OS={{ k8s.eject.iso }} @@ -230,6 +237,34 @@ write_files: done fi + - path: /etc/kubernetes/kubeadm-config.yaml + permissions: '0644' + owner: root:root + content: | + apiVersion: kubeadm.k8s.io/v1beta3 + kind: ClusterConfiguration + apiServer: + certSANs: + {{ k8s_control.server_ips }} + controlPlaneEndpoint: {{ k8s_control.server_ip }}:{{ k8s.api_server_port }} + etcd: + external: + endpoints: + {{ etcd.etcd_endpoint_list }} + --- + apiVersion: kubeadm.k8s.io/v1beta3 + kind: InitConfiguration + bootstrapTokens: + - token: "{{ k8s_control_node.cluster.token }}" + ttl: "0" + nodeRegistration: + criSocket: /run/containerd/containerd.sock + localAPIEndpoint: + advertiseAddress: {{ k8s_control.server_ip }} + bindPort: {{ k8s.api_server_port }} + certificateKey: {{ k8s_control.certificate_key }} + + - path: /opt/bin/deploy-kube-system permissions: '0700' owner: root:root @@ -245,6 +280,8 @@ write_files: export PATH=$PATH:/opt/bin fi + EXTERNAL_ETCD_NODES={{ etcd.unstacked_etcd }} + EXTERNAL_CNI_PLUGIN={{ k8s.external.cni.plugin }} MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 crucial_cmd_attempts=1 while true; do @@ -254,7 +291,11 @@ write_files: fi retval=0 set +e - kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + if [[ ${EXTERNAL_ETCD_NODES} == true ]]; then + kubeadm init --config /etc/kubernetes/kubeadm-config.yaml --upload-certs + else + kubeadm init --token {{ k8s_control_node.cluster.token }} --token-ttl 0 {{ k8s_control_node.cluster.initargs }} --cri-socket /run/containerd/containerd.sock + fi retval=$? set -e if [ $retval -eq 0 ]; then @@ -282,7 +323,9 @@ write_files: if [ -d "$K8S_CONFIG_SCRIPTS_COPY_DIR" ]; then ### Network, dashboard configs available offline ### echo "Offline configs are available!" - /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + if [[ ${EXTERNAL_CNI_PLUGIN} == false ]]; then + /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/network.yaml + fi /opt/bin/kubectl apply -f ${K8S_CONFIG_SCRIPTS_COPY_DIR}/dashboard.yaml rm -rf "${K8S_CONFIG_SCRIPTS_COPY_DIR}" else @@ -297,6 +340,7 @@ write_files: sudo touch /home/cloud/success echo "true" > /home/cloud/success + {% if registry is defined %} - path: /opt/bin/setup-containerd permissions: '0755' owner: root:root @@ -314,6 +358,7 @@ write_files: echo "Restarting containerd service" systemctl daemon-reload systemctl restart containerd + {% endif %} - path: /etc/systemd/system/deploy-kube-system.service permissions: '0755' diff --git a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml index ac80e8576ff..fd9617e69de 100644 --- a/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml +++ b/plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml @@ -40,8 +40,8 @@ write_files: sysctl net.ipv4.conf.default.arp_ignore=0 sysctl net.ipv4.conf.all.arp_announce=0 sysctl net.ipv4.conf.all.arp_ignore=0 - sysctl net.ipv4.conf.eth0.arp_announce=0 - sysctl net.ipv4.conf.eth0.arp_ignore=0 + sysctl net.ipv4.conf.eth0.arp_announce=0 || sysctl net.ipv4.conf.ens35.arp_announce=0 || true + sysctl net.ipv4.conf.eth0.arp_ignore=0 || sysctl net.ipv4.conf.ens35.arp_ignore=0 || true sed -i "s/net.ipv4.conf.default.arp_announce =.*$/net.ipv4.conf.default.arp_announce = 0/" /etc/sysctl.conf sed -i "s/net.ipv4.conf.default.arp_ignore =.*$/net.ipv4.conf.default.arp_ignore = 0/" /etc/sysctl.conf sed -i "s/net.ipv4.conf.all.arp_announce =.*$/net.ipv4.conf.all.arp_announce = 0/" /etc/sysctl.conf @@ -53,8 +53,14 @@ write_files: ATTEMPT_ONLINE_INSTALL=false setup_complete=false - OFFLINE_INSTALL_ATTEMPT_SLEEP=30 - MAX_OFFLINE_INSTALL_ATTEMPTS=40 + OFFLINE_INSTALL_ATTEMPT_SLEEP={{ k8s.install.wait.time }} + MAX_OFFLINE_INSTALL_ATTEMPTS={{ k8s.install.reattempts.count }} + if [[ -z $OFFLINE_INSTALL_ATTEMPT_SLEEP || $OFFLINE_INSTALL_ATTEMPT_SLEEP -eq 0 ]]; then + OFFLINE_INSTALL_ATTEMPT_SLEEP=30 + fi + if [[ -z $MAX_OFFLINE_INSTALL_ATTEMPTS || $MAX_OFFLINE_INSTALL_ATTEMPTS -eq 0 ]]; then + MAX_OFFLINE_INSTALL_ATTEMPTS=40 + fi offline_attempts=1 MAX_SETUP_CRUCIAL_CMD_ATTEMPTS=3 EJECT_ISO_FROM_OS={{ k8s.eject.iso }} @@ -87,6 +93,21 @@ write_files: fi fi done <<< "$output" + else + ### Download from VR ### + ROUTER_IP="{{ k8s.vr.iso.mounted.ip }}" + if [ "$ROUTER_IP" != "" ]; then + echo "Downloading CKS binaries from the VR $ROUTER_IP" + if [ ! -d "${ISO_MOUNT_DIR}" ]; then + mkdir "${ISO_MOUNT_DIR}" + fi + ### Download from ROUTER_IP/cks-iso into ISO_MOUNT_DIR + AUX_DOWNLOAD_DIR=/aux-dwnld + mkdir -p $AUX_DOWNLOAD_DIR + wget -r -R "index.html*" $ROUTER_IP/cks-iso -P $AUX_DOWNLOAD_DIR || echo 'Cannot download some files from virtual router' + mv $AUX_DOWNLOAD_DIR/$ROUTER_IP/cks-iso/* $ISO_MOUNT_DIR + rm -rf $AUX_DOWNLOAD_DIR + fi fi if [ -d "$BINARIES_DIR" ]; then break diff --git a/test/selenium/lib/initialize.py b/plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster similarity index 53% rename from test/selenium/lib/initialize.py rename to plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster index 4e451838dfa..f852a0fd4dd 100644 --- a/test/selenium/lib/initialize.py +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/remove-node-from-cluster @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,33 +15,29 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -''' -This will help pass webdriver (Browser instance) across our test cases. -''' +export PATH=$PATH:/opt/bin +node_name=$1 +node_type=$2 +operation=$3 - -from selenium import webdriver -import sys - -DRIVER = None -MS_ip = None - - -def getOrCreateWebdriver(): - global DRIVER - DRIVER = DRIVER or webdriver.PhantomJS('phantomjs') # phantomjs executable must be in PATH. - return DRIVER - - -def getMSip(): - global MS_ip - if len(sys.argv) >= 3: - sys.exit("Only One argument is required .. Enter your Management Server IP") - - if len(sys.argv) == 1: - sys.exit("Atleast One argument is required .. Enter your Management Server IP") - - for arg in sys.argv[1:]: - MS_ip = arg - return MS_ip +if [ $operation == "remove" ]; then + if [ $node_type == "control" ]; then + # get the specific node + kubectl get nodes $node_name >/dev/null 2>&1 + if [[ $(echo $?) -eq 1 ]]; then + echo "No node with name $node_name present in the cluster, exiting..." + exit 0 + else + # Drain the node + kubectl drain $node_name --delete-local-data --force --ignore-daemonsets + fi + else + kubeadm reset -f + fi +else + sudo mkdir -p /home/cloud/.kube + sudo cp /root/.kube/config /home/cloud/.kube/ + sudo chown -R cloud:cloud /home/cloud/.kube + kubectl delete node $node_name +fi diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh index 480b002ef17..a947d508436 100755 --- a/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/upgrade-kubernetes.sh @@ -18,7 +18,7 @@ # Version 1.14 and below needs extra flags with kubeadm upgrade node if [ $# -lt 4 ]; then - echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_CONTROL_NODE IS_OLD_VERSION IS_EJECT_ISO" + echo "Invalid input. Valid usage: ./upgrade-kubernetes.sh UPGRADE_VERSION IS_CONTROL_NODE IS_OLD_VERSION IS_EJECT_ISO IS_EXTERNAL_CNI" echo "eg: ./upgrade-kubernetes.sh 1.16.3 true false false" exit 1 fi @@ -35,6 +35,10 @@ EJECT_ISO_FROM_OS=false if [ $# -gt 3 ]; then EJECT_ISO_FROM_OS="${4}" fi +EXTERNAL_CNI=false +if [ $# -gt 4 ]; then + EXTERNAL_CNI="${5}" +fi export PATH=$PATH:/opt/bin if [[ "$PATH" != *:/usr/sbin && "$PATH" != *:/usr/sbin:* ]]; then @@ -144,7 +148,9 @@ if [ -d "$BINARIES_DIR" ]; then systemctl restart kubelet if [ "${IS_MAIN_CONTROL}" == 'true' ]; then - /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml + if [[ ${EXTERNAL_CNI} == true ]]; then + /opt/bin/kubectl apply -f ${BINARIES_DIR}/network.yaml + fi /opt/bin/kubectl apply -f ${BINARIES_DIR}/dashboard.yaml fi diff --git a/plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node b/plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node new file mode 100644 index 00000000000..e28614e05af --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/main/resources/script/validate-cks-node @@ -0,0 +1,45 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +OS=`awk -F= '/^NAME/{print $2}' /etc/os-release` +REQUIRED_PACKAGES=(cloud-init cloud-guest-utils conntrack apt-transport-https ca-certificates curl gnupg gnupg-agent \ + software-properties-common gnupg lsb-release python3-json-pointer python3-jsonschema cloud-init containerd.io) +declare -a MISSING_PACKAGES +if [[ $OS == *"Ubuntu"* || $OS == *"Debian"* ]]; then + for package in ${REQUIRED_PACKAGES[@]}; do + dpkg -s $package >/dev/null 2>&1 + if [ $? -eq 1 ]; then + MISSING_PACKAGES+="$package" + fi + done +else + for package in ${REQUIRED_PACKAGES[@]}; do + rpm -qa | grep $package >/dev/null 2>&1 + if [ $? -eq 1 ]; then + MISSING_PACKAGES[${#MISSING_PACKAGES[@]}]=$package + fi + done +fi + +echo ${#MISSING_PACKAGES[@]} +if (( ${#MISSING_PACKAGES[@]} )); then + echo "Following packages are missing in the node template: ${MISSING_PACKAGES[@]}" + exit 1 +else + echo 0 +fi diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java new file mode 100644 index 00000000000..298f1dfbcd6 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImplTest.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster; + +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.vm.VmDetailConstants; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterHelperImplTest { + + @Mock + private ServiceOfferingDao serviceOfferingDao; + @Mock + private ServiceOfferingVO workerServiceOffering; + @Mock + private ServiceOfferingVO controlServiceOffering; + @Mock + private ServiceOfferingVO etcdServiceOffering; + + private static final String workerNodesOfferingId = UUID.randomUUID().toString(); + private static final String controlNodesOfferingId = UUID.randomUUID().toString(); + private static final String etcdNodesOfferingId = UUID.randomUUID().toString(); + private static final Long workerOfferingId = 1L; + private static final Long controlOfferingId = 2L; + private static final Long etcdOfferingId = 3L; + + private final KubernetesServiceHelperImpl helper = new KubernetesServiceHelperImpl(); + + @Before + public void setUp() { + helper.serviceOfferingDao = serviceOfferingDao; + Mockito.when(serviceOfferingDao.findByUuid(workerNodesOfferingId)).thenReturn(workerServiceOffering); + Mockito.when(serviceOfferingDao.findByUuid(controlNodesOfferingId)).thenReturn(controlServiceOffering); + Mockito.when(serviceOfferingDao.findByUuid(etcdNodesOfferingId)).thenReturn(etcdServiceOffering); + Mockito.when(workerServiceOffering.getId()).thenReturn(workerOfferingId); + Mockito.when(controlServiceOffering.getId()).thenReturn(controlOfferingId); + Mockito.when(etcdServiceOffering.getId()).thenReturn(etcdOfferingId); + } + + @Test + public void testIsValidNodeTypeEmptyNodeType() { + Assert.assertFalse(helper.isValidNodeType(null)); + } + + @Test + public void testIsValidNodeTypeInvalidNodeType() { + String nodeType = "invalidNodeType"; + Assert.assertFalse(helper.isValidNodeType(nodeType)); + } + + @Test + public void testIsValidNodeTypeValidNodeTypeLowercase() { + String nodeType = KubernetesServiceHelper.KubernetesClusterNodeType.WORKER.name().toLowerCase(); + Assert.assertTrue(helper.isValidNodeType(nodeType)); + } + + private Map createMapEntry(KubernetesServiceHelper.KubernetesClusterNodeType nodeType, + String nodeTypeOfferingUuid) { + Map map = new HashMap<>(); + map.put(VmDetailConstants.CKS_NODE_TYPE, nodeType.name().toLowerCase()); + map.put(VmDetailConstants.OFFERING, nodeTypeOfferingUuid); + return map; + } + + @Test + public void testNodeOfferingMap() { + Map> serviceOfferingNodeTypeMap = new HashMap<>(); + Map firstMap = createMapEntry(WORKER, workerNodesOfferingId); + Map secondMap = createMapEntry(CONTROL, controlNodesOfferingId); + serviceOfferingNodeTypeMap.put("map1", firstMap); + serviceOfferingNodeTypeMap.put("map2", secondMap); + Map map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + Assert.assertNotNull(map); + Assert.assertEquals(2, map.size()); + Assert.assertTrue(map.containsKey(WORKER.name()) && map.containsKey(CONTROL.name())); + Assert.assertEquals(workerOfferingId, map.get(WORKER.name())); + Assert.assertEquals(controlOfferingId, map.get(CONTROL.name())); + } + + @Test + public void testNodeOfferingMapNullMap() { + Map map = helper.getServiceOfferingNodeTypeMap(null); + Assert.assertTrue(map.isEmpty()); + } + + @Test + public void testNodeOfferingMapEtcdNodes() { + Map> serviceOfferingNodeTypeMap = new HashMap<>(); + Map firstMap = createMapEntry(ETCD, etcdNodesOfferingId); + serviceOfferingNodeTypeMap.put("map1", firstMap); + Map map = helper.getServiceOfferingNodeTypeMap(serviceOfferingNodeTypeMap); + Assert.assertNotNull(map); + Assert.assertEquals(1, map.size()); + Assert.assertTrue(map.containsKey(ETCD.name())); + Assert.assertEquals(etcdOfferingId, map.get(ETCD.name())); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryCompletenessInvalidParameters() { + helper.checkNodeTypeOfferingEntryCompleteness(WORKER.name(), null); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesInvalidNodeType() { + String invalidNodeType = "invalidNodeTypeName"; + helper.checkNodeTypeOfferingEntryValues(invalidNodeType, workerServiceOffering, workerNodesOfferingId); + } + + @Test(expected = InvalidParameterValueException.class) + public void testCheckNodeTypeOfferingEntryValuesEmptyOffering() { + String nodeType = WORKER.name(); + helper.checkNodeTypeOfferingEntryValues(nodeType, null, workerNodesOfferingId); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java index f64c467b71b..57715cc9309 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImplTest.java @@ -27,16 +27,21 @@ import com.cloud.exception.PermissionDeniedException; import com.cloud.kubernetes.cluster.actionworkers.KubernetesClusterActionWorker; import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao; import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.network.Network; import com.cloud.network.dao.FirewallRulesDao; import com.cloud.network.rules.FirewallRule; import com.cloud.network.rules.FirewallRuleVO; import com.cloud.network.vpc.NetworkACL; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; import com.cloud.user.User; +import com.cloud.utils.Pair; import com.cloud.utils.net.NetUtils; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.dao.VMInstanceDao; @@ -45,6 +50,7 @@ import org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachi import org.apache.cloudstack.api.command.user.kubernetes.cluster.RemoveVirtualMachinesFromKubernetesClusterCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.commons.collections.MapUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -60,7 +66,14 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.ETCD; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.WORKER; @RunWith(MockitoJUnitRunner.class) public class KubernetesClusterManagerImplTest { @@ -86,6 +99,9 @@ public class KubernetesClusterManagerImplTest { @Mock private AccountManager accountManager; + @Mock + private ServiceOfferingDao serviceOfferingDao; + @Spy @InjectMocks KubernetesClusterManagerImpl kubernetesClusterManager; @@ -293,4 +309,117 @@ public class KubernetesClusterManagerImplTest { Mockito.when(kubernetesClusterDao.findById(Mockito.anyLong())).thenReturn(cluster); Assert.assertTrue(kubernetesClusterManager.removeVmsFromCluster(cmd).size() > 0); } + + @Test + public void testValidateServiceOfferingNodeType() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), 2L); + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering); + Mockito.when(serviceOffering.isDynamic()).thenReturn(false); + Mockito.when(serviceOffering.getCpu()).thenReturn(2); + Mockito.when(serviceOffering.getRamSize()).thenReturn(2048); + KubernetesSupportedVersion version = Mockito.mock(KubernetesSupportedVersion.class); + Mockito.when(version.getMinimumCpu()).thenReturn(2); + Mockito.when(version.getMinimumRamSize()).thenReturn(2048); + kubernetesClusterManager.validateServiceOfferingForNode(map, 1L, WORKER.name(), null, version); + Mockito.verify(kubernetesClusterManager).validateServiceOffering(serviceOffering, version); + } + + @Test(expected = InvalidParameterValueException.class) + public void testValidateServiceOfferingNodeTypeInvalidOffering() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), 2L); + ServiceOfferingVO serviceOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(serviceOffering); + Mockito.when(serviceOffering.isDynamic()).thenReturn(true); + kubernetesClusterManager.validateServiceOfferingForNode(map, 1L, WORKER.name(), null, null); + } + + @Test + public void testClusterCapacity() { + long workerOfferingId = 1L; + long controlOfferingId = 2L; + long workerCount = 2L; + long controlCount = 2L; + + int workerOfferingCpus = 4; + int workerOfferingMemory = 4096; + int controlOfferingCpus = 2; + int controlOfferingMemory = 2048; + + Map map = Map.of(WORKER.name(), workerOfferingId, CONTROL.name(), controlOfferingId); + Map nodeCount = Map.of(WORKER.name(), workerCount, CONTROL.name(), controlCount); + + ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(workerOfferingId)).thenReturn(workerOffering); + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(controlOfferingId)).thenReturn(controlOffering); + Mockito.when(workerOffering.getCpu()).thenReturn(workerOfferingCpus); + Mockito.when(workerOffering.getRamSize()).thenReturn(workerOfferingMemory); + Mockito.when(controlOffering.getCpu()).thenReturn(controlOfferingCpus); + Mockito.when(controlOffering.getRamSize()).thenReturn(controlOfferingMemory); + + Pair pair = kubernetesClusterManager.calculateClusterCapacity(map, nodeCount, 1L); + Long expectedCpu = (workerOfferingCpus * workerCount) + (controlOfferingCpus * controlCount); + Long expectedMemory = (workerOfferingMemory * workerCount) + (controlOfferingMemory * controlCount); + Assert.assertEquals(expectedCpu, pair.first()); + Assert.assertEquals(expectedMemory, pair.second()); + } + + @Test + public void testIsAnyNodeOfferingEmptyNullMap() { + Assert.assertTrue(kubernetesClusterManager.isAnyNodeOfferingEmpty(null)); + } + + @Test + public void testIsAnyNodeOfferingEmptyNullValue() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), null); + map.put(ETCD.name(), 2L); + Assert.assertTrue(kubernetesClusterManager.isAnyNodeOfferingEmpty(map)); + } + + @Test + public void testIsAnyNodeOfferingEmpty() { + Map map = new HashMap<>(); + map.put(WORKER.name(), 1L); + map.put(CONTROL.name(), 2L); + Assert.assertFalse(kubernetesClusterManager.isAnyNodeOfferingEmpty(map)); + } + + @Test + public void testCreateNodeTypeToServiceOfferingMapNullMap() { + KubernetesClusterVO clusterVO = Mockito.mock(KubernetesClusterVO.class); + Mockito.when(clusterVO.getServiceOfferingId()).thenReturn(1L); + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(offering); + Map mapping = kubernetesClusterManager.createNodeTypeToServiceOfferingMap(new HashMap<>(), null, clusterVO); + Assert.assertFalse(MapUtils.isEmpty(mapping)); + Assert.assertTrue(mapping.containsKey(DEFAULT.name())); + Assert.assertEquals(offering, mapping.get(DEFAULT.name())); + } + + @Test + public void testCreateNodeTypeToServiceOfferingMap() { + Map idsMap = new HashMap<>(); + long workerOfferingId = 1L; + long controlOfferingId = 2L; + idsMap.put(WORKER.name(), workerOfferingId); + idsMap.put(CONTROL.name(), controlOfferingId); + + ServiceOfferingVO workerOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(workerOfferingId)).thenReturn(workerOffering); + ServiceOfferingVO controlOffering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(serviceOfferingDao.findById(controlOfferingId)).thenReturn(controlOffering); + + Map mapping = kubernetesClusterManager.createNodeTypeToServiceOfferingMap(idsMap, null, null); + Assert.assertEquals(2, mapping.size()); + Assert.assertTrue(mapping.containsKey(WORKER.name()) && mapping.containsKey(CONTROL.name())); + Assert.assertEquals(workerOffering, mapping.get(WORKER.name())); + Assert.assertEquals(controlOffering, mapping.get(CONTROL.name())); + } } diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java new file mode 100644 index 00000000000..847c8bd6d29 --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterScaleWorkerTest.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.offering.ServiceOffering; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.utils.Pair; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.dao.UserVmDao; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.List; + +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.DEFAULT; +import static com.cloud.kubernetes.cluster.KubernetesServiceHelper.KubernetesClusterNodeType.CONTROL; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterScaleWorkerTest { + + @Mock + private KubernetesCluster kubernetesCluster; + @Mock + private KubernetesClusterManagerImpl clusterManager; + @Mock + private ServiceOfferingDao serviceOfferingDao; + @Mock + private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + @Mock + private UserVmDao userVmDao; + + private KubernetesClusterScaleWorker worker; + + private static final Long defaultOfferingId = 1L; + + @Before + public void setUp() { + worker = new KubernetesClusterScaleWorker(kubernetesCluster, clusterManager); + worker.serviceOfferingDao = serviceOfferingDao; + worker.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao; + worker.userVmDao = userVmDao; + } + + @Test + public void testCalculateNewClusterCountAndCapacityAllNodesScaleSize() { + long controlNodes = 3L; + long etcdNodes = 2L; + Mockito.when(kubernetesCluster.getControlNodeCount()).thenReturn(controlNodes); + Mockito.when(kubernetesCluster.getEtcdNodeCount()).thenReturn(etcdNodes); + + ServiceOffering newOffering = Mockito.mock(ServiceOffering.class); + int newCores = 4; + int newMemory = 4096; + Mockito.when(newOffering.getCpu()).thenReturn(newCores); + Mockito.when(newOffering.getRamSize()).thenReturn(newMemory); + + long newWorkerSize = 4L; + Pair newClusterCapacity = worker.calculateNewClusterCountAndCapacity(newWorkerSize, DEFAULT, newOffering); + + long expectedCores = (newCores * newWorkerSize) + (newCores * controlNodes) + (newCores * etcdNodes); + long expectedMemory = (newMemory * newWorkerSize) + (newMemory * controlNodes) + (newMemory * etcdNodes); + Assert.assertEquals(expectedCores, newClusterCapacity.first().longValue()); + Assert.assertEquals(expectedMemory, newClusterCapacity.second().longValue()); + } + + @Test + public void testCalculateNewClusterCountAndCapacityNodeTypeScaleControlOffering() { + long controlNodes = 2L; + long kubernetesClusterId = 10L; + Mockito.when(kubernetesCluster.getId()).thenReturn(kubernetesClusterId); + Mockito.when(kubernetesCluster.getControlNodeCount()).thenReturn(controlNodes); + + ServiceOfferingVO existingOffering = Mockito.mock(ServiceOfferingVO.class); + int existingCores = 2; + int existingMemory = 2048; + Mockito.when(existingOffering.getCpu()).thenReturn(existingCores); + Mockito.when(existingOffering.getRamSize()).thenReturn(existingMemory); + int remainingClusterCpu = 8; + int remainingClusterMemory = 12288; + Mockito.when(kubernetesCluster.getCores()).thenReturn(remainingClusterCpu + (controlNodes * existingCores)); + Mockito.when(kubernetesCluster.getMemory()).thenReturn(remainingClusterMemory + (controlNodes * existingMemory)); + + Mockito.when(serviceOfferingDao.findById(1L)).thenReturn(existingOffering); + + ServiceOfferingVO newOffering = Mockito.mock(ServiceOfferingVO.class); + int newCores = 4; + int newMemory = 2048; + Mockito.when(newOffering.getCpu()).thenReturn(newCores); + Mockito.when(newOffering.getRamSize()).thenReturn(newMemory); + + KubernetesClusterVmMapVO controlNodeVM1 = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(controlNodeVM1.getVmId()).thenReturn(10L); + UserVmVO userVmVO = Mockito.mock(UserVmVO.class); + Mockito.when(userVmVO.getServiceOfferingId()).thenReturn(defaultOfferingId); + Mockito.when(userVmDao.findById(10L)).thenReturn(userVmVO); + Mockito.when(kubernetesClusterVmMapDao.listByClusterIdAndVmType(kubernetesClusterId, CONTROL)).thenReturn(List.of(controlNodeVM1)); + Pair newClusterCapacity = worker.calculateNewClusterCountAndCapacity(null, CONTROL, newOffering); + + long expectedCores = remainingClusterCpu + (controlNodes * newCores); + long expectedMemory = remainingClusterMemory + (controlNodes * newMemory); + Assert.assertEquals(expectedCores, newClusterCapacity.first().longValue()); + Assert.assertEquals(expectedMemory, newClusterCapacity.second().longValue()); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java new file mode 100644 index 00000000000..ff8f875d06e --- /dev/null +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.kubernetes.cluster.actionworkers; + +import com.cloud.kubernetes.cluster.KubernetesCluster; +import com.cloud.kubernetes.cluster.KubernetesClusterManagerImpl; +import com.cloud.kubernetes.cluster.KubernetesClusterVmMapVO; +import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; +import com.cloud.kubernetes.version.KubernetesSupportedVersion; +import com.cloud.uservm.UserVm; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +@RunWith(MockitoJUnitRunner.class) +public class KubernetesClusterUpgradeWorkerTest { + + @Mock + private KubernetesCluster kubernetesCluster; + @Mock + private KubernetesSupportedVersion kubernetesSupportedVersion; + @Mock + private KubernetesClusterManagerImpl clusterManager; + @Mock + private KubernetesClusterVmMapDao kubernetesClusterVmMapDao; + + private KubernetesClusterUpgradeWorker worker; + + @Before + public void setUp() { + String[] keys = {}; + worker = new KubernetesClusterUpgradeWorker(kubernetesCluster, kubernetesSupportedVersion, clusterManager, keys); + worker.kubernetesClusterVmMapDao = kubernetesClusterVmMapDao; + } + + @Test + public void testFilterOutManualUpgradeNodesFromClusterUpgrade() { + long controlNodeId = 1L; + long workerNode1Id = 2L; + long workerNode2Id = 3L; + UserVm controlNode = Mockito.mock(UserVm.class); + Mockito.when(controlNode.getId()).thenReturn(controlNodeId); + UserVm workerNode1 = Mockito.mock(UserVm.class); + Mockito.when(workerNode1.getId()).thenReturn(workerNode1Id); + UserVm workerNode2 = Mockito.mock(UserVm.class); + Mockito.when(workerNode2.getId()).thenReturn(workerNode2Id); + KubernetesClusterVmMapVO controlNodeMap = Mockito.mock(KubernetesClusterVmMapVO.class); + KubernetesClusterVmMapVO workerNode1Map = Mockito.mock(KubernetesClusterVmMapVO.class); + KubernetesClusterVmMapVO workerNode2Map = Mockito.mock(KubernetesClusterVmMapVO.class); + Mockito.when(workerNode2Map.isManualUpgrade()).thenReturn(true); + Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(controlNodeId)).thenReturn(controlNodeMap); + Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(workerNode1Id)).thenReturn(workerNode1Map); + Mockito.when(kubernetesClusterVmMapDao.getClusterMapFromVmId(workerNode2Id)).thenReturn(workerNode2Map); + worker.clusterVMs = Arrays.asList(controlNode, workerNode1, workerNode2); + worker.filterOutManualUpgradeNodesFromClusterUpgrade(); + Assert.assertEquals(2, worker.clusterVMs.size()); + List ids = worker.clusterVMs.stream().map(UserVm::getId).collect(Collectors.toList()); + Assert.assertTrue(ids.contains(controlNodeId) && ids.contains(workerNode1Id)); + Assert.assertFalse(ids.contains(workerNode2Id)); + } +} diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java index 31363dbd1a1..329f9b0e42a 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/utils/KubernetesClusterUtilTest.java @@ -27,14 +27,14 @@ public class KubernetesClusterUtilTest { private void executeThrowAndTestVersionMatch() { Pair resultPair = null; - boolean result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0"); - Assert.assertFalse(result); + Pair result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0"); + Assert.assertFalse(result.first()); } private void executeAndTestVersionMatch(boolean status, String response, boolean expectedResult) { Pair resultPair = new Pair<>(status, response); - boolean result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0"); - Assert.assertEquals(expectedResult, result); + Pair result = KubernetesClusterUtil.clusterNodeVersionMatches(resultPair, "1.24.0"); + Assert.assertEquals(expectedResult, result.first()); } @Test diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/CancelMaintenanceCmd.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/CancelMaintenanceCmd.java index a0f091ef1e4..ab3f900b693 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/CancelMaintenanceCmd.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/CancelMaintenanceCmd.java @@ -18,12 +18,15 @@ package org.apache.cloudstack.api.command; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import com.cloud.user.Account; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.ManagementServerMaintenanceResponse; import org.apache.cloudstack.acl.RoleType; +import org.apache.commons.lang3.BooleanUtils; @APICommand(name = CancelMaintenanceCmd.APINAME, description = "Cancels maintenance of the management server", @@ -36,6 +39,13 @@ public class CancelMaintenanceCmd extends BaseMSMaintenanceActionCmd { public static final String APINAME = "cancelMaintenance"; + @Parameter(name = ApiConstants.REBALANCE, type = CommandType.BOOLEAN, description = "Rebalance agents (applicable for indirect agents, ensure the settings 'host' and 'indirect.agent.lb.algorithm' are properly configured) after cancelling maintenance, default is true") + private Boolean rebalance; + + public boolean getRebalance() { + return BooleanUtils.toBooleanDefaultIfNull(rebalance, true); + } + @Override public String getCommandName() { return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/PrepareForMaintenanceCmd.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/PrepareForMaintenanceCmd.java index 3c036c4c35f..2b63b28e0c5 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/PrepareForMaintenanceCmd.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/api/command/PrepareForMaintenanceCmd.java @@ -26,6 +26,7 @@ import com.cloud.user.Account; import org.apache.cloudstack.api.response.ManagementServerMaintenanceResponse; import org.apache.cloudstack.acl.RoleType; +import org.apache.commons.lang3.BooleanUtils; @APICommand(name = PrepareForMaintenanceCmd.APINAME, description = "Prepares management server for maintenance by preventing new jobs from being accepted after completion of active jobs and migrating the agents", @@ -40,6 +41,9 @@ public class PrepareForMaintenanceCmd extends BaseMSMaintenanceActionCmd { " when this is not set, already configured algorithm from setting 'indirect.agent.lb.algorithm' is considered") private String algorithm; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, description = "Force management server to maintenance after the maintenance window timeout, default is false") + private Boolean forced; + public String getAlgorithm() { return algorithm; } @@ -48,6 +52,10 @@ public class PrepareForMaintenanceCmd extends BaseMSMaintenanceActionCmd { this.algorithm = algorithm; } + public boolean isForced() { + return BooleanUtils.toBooleanDefaultIfNull(forced, false); + } + @Override public String getCommandName() { return APINAME.toLowerCase() + BaseCmd.RESPONSE_SUFFIX; diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java index 3af19164cc9..b7b68b065c0 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManager.java @@ -40,6 +40,15 @@ public interface ManagementServerMaintenanceManager { ConfigKey.Scope.Global, null); + ConfigKey ManagementServerMaintenanceIgnoreMaintenanceHosts = new ConfigKey<>(Boolean.class, + "management.server.maintenance.ignore.maintenance.hosts", + "Advanced", + String.valueOf(Boolean.FALSE), + "Host in Maintenance state can sometimes block Management Server to go to Maintenance; this setting skips Host(s) in Maintenance state during Management Server Maintenance, default: false.", + true, + ConfigKey.Scope.Global, + null); + void registerListener(ManagementServerMaintenanceListener listener); void unregisterListener(ManagementServerMaintenanceListener listener); @@ -76,14 +85,14 @@ public interface ManagementServerMaintenanceManager { // Indicates whether the current management server is preparing to maintenance boolean isPreparingForMaintenance(); - void resetPreparingForMaintenance(); + void resetMaintenanceParams(); long getMaintenanceStartTime(); String getLbAlgorithm(); // Prepares the current management server for maintenance by migrating the agents and not accepting any more async jobs - void prepareForMaintenance(String lbAlorithm); + void prepareForMaintenance(String lbAlorithm, boolean forced); // Cancels maintenance of the current management server void cancelMaintenance(); diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java index fcfa32d6ce8..16cf14e1fb1 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImpl.java @@ -26,7 +26,9 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; +import com.cloud.resource.ResourceState; import org.apache.cloudstack.agent.lb.IndirectAgentLB; +import org.apache.cloudstack.agent.lb.IndirectAgentLBServiceImpl; import org.apache.cloudstack.api.command.CancelMaintenanceCmd; import org.apache.cloudstack.api.command.CancelShutdownCmd; import org.apache.cloudstack.api.command.PrepareForMaintenanceCmd; @@ -39,6 +41,7 @@ import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.management.ManagementServerHost.State; import org.apache.cloudstack.maintenance.command.CancelMaintenanceManagementServerHostCommand; import org.apache.cloudstack.maintenance.command.CancelShutdownManagementServerHostCommand; @@ -196,13 +199,20 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen return preparingForShutdown; } + private void resetShutdownParams() { + logger.debug("Resetting shutdown params"); + preparingForShutdown = false; + shutdownTriggered = false; + } + @Override public boolean isPreparingForMaintenance() { return preparingForMaintenance; } @Override - public void resetPreparingForMaintenance() { + public void resetMaintenanceParams() { + logger.debug("Resetting maintenance params"); preparingForMaintenance = false; maintenanceStartTime = 0; lbAlgorithm = null; @@ -235,6 +245,11 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } this.shutdownTriggered = true; prepareForShutdown(true); + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + throw new CloudRuntimeException("Invalid node id for the management server"); + } + msHostDao.updateState(msHost.getId(), State.ShuttingDown); } private void prepareForShutdown(boolean postTrigger) { @@ -251,29 +266,38 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen this.preparingForShutdown = true; jobManager.disableAsyncJobs(); - waitForPendingJobs(); + waitForPendingJobs(false); } @Override public void prepareForShutdown() { prepareForShutdown(false); + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + throw new CloudRuntimeException("Invalid node id for the management server"); + } + msHostDao.updateState(msHost.getId(), State.PreparingForShutDown); } @Override public void cancelShutdown() { - if (!this.preparingForShutdown) { + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + throw new CloudRuntimeException("Invalid node id for the management server"); + } + if (!this.preparingForShutdown && !(State.PreparingForShutDown.equals(msHost.getState()) || State.ReadyToShutDown.equals(msHost.getState()))) { throw new CloudRuntimeException("Shutdown has not been triggered"); } - this.preparingForShutdown = false; - this.shutdownTriggered = false; - resetPreparingForMaintenance(); + resetShutdownParams(); + resetMaintenanceParams(); jobManager.enableAsyncJobs(); cancelWaitForPendingJobs(); + msHostDao.updateState(msHost.getId(), State.Up); } @Override - public void prepareForMaintenance(String lbAlorithm) { + public void prepareForMaintenance(String lbAlorithm, boolean forced) { if (this.preparingForShutdown) { throw new CloudRuntimeException("Shutdown has already been triggered, cancel shutdown and try again"); } @@ -281,41 +305,57 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen if (this.preparingForMaintenance) { throw new CloudRuntimeException("Maintenance has already been initiated"); } + + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + throw new CloudRuntimeException("Invalid node id for the management server"); + } this.preparingForMaintenance = true; this.maintenanceStartTime = System.currentTimeMillis(); this.lbAlgorithm = lbAlorithm; jobManager.disableAsyncJobs(); onPreparingForMaintenance(); - waitForPendingJobs(); + waitForPendingJobs(forced); + msHostDao.updateState(msHost.getId(), State.PreparingForMaintenance); } @Override public void cancelMaintenance() { - if (!this.preparingForMaintenance) { + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + throw new CloudRuntimeException("Invalid node id for the management server"); + } + if (!this.preparingForMaintenance && !(State.Maintenance.equals(msHost.getState()) || State.PreparingForMaintenance.equals(msHost.getState()))) { throw new CloudRuntimeException("Maintenance has not been initiated"); } - resetPreparingForMaintenance(); - this.preparingForShutdown = false; - this.shutdownTriggered = false; + resetMaintenanceParams(); + resetShutdownParams(); jobManager.enableAsyncJobs(); cancelWaitForPendingJobs(); - ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); - if (msHost != null) { - if (State.PreparingForMaintenance.equals(msHost.getState())) { - onCancelPreparingForMaintenance(); - } - if (State.Maintenance.equals(msHost.getState())) { - onCancelMaintenance(); - } + msHostDao.updateState(msHost.getId(), State.Up); + ScheduledExecutorService cancelMaintenanceService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("CancelMaintenance-Job")); + cancelMaintenanceService.schedule(() -> { + cancelMaintenanceTask(msHost.getState()); + }, 0, TimeUnit.SECONDS); + cancelMaintenanceService.shutdown(); + } + + private void cancelMaintenanceTask(ManagementServerHost.State msState) { + if (State.PreparingForMaintenance.equals(msState)) { + onCancelPreparingForMaintenance(); + } + if (State.Maintenance.equals(msState)) { + onCancelMaintenance(); } } - private void waitForPendingJobs() { + private void waitForPendingJobs(boolean forceMaintenance) { cancelWaitForPendingJobs(); pendingJobsCheckTask = Executors.newScheduledThreadPool(1, new NamedThreadFactory("PendingJobsCheck")); long pendingJobsCheckDelayInSecs = 1L; // 1 sec long pendingJobsCheckPeriodInSecs = 3L; // every 3 secs, check more frequently for pending jobs - pendingJobsCheckTask.scheduleAtFixedRate(new CheckPendingJobsTask(this), pendingJobsCheckDelayInSecs, pendingJobsCheckPeriodInSecs, TimeUnit.SECONDS); + boolean ignoreMaintenanceHosts = ManagementServerMaintenanceIgnoreMaintenanceHosts.value(); + pendingJobsCheckTask.scheduleAtFixedRate(new CheckPendingJobsTask(this, ignoreMaintenanceHosts, forceMaintenance), pendingJobsCheckDelayInSecs, pendingJobsCheckPeriodInSecs, TimeUnit.SECONDS); } @Override @@ -349,7 +389,6 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen cmds[0] = new PrepareForShutdownManagementServerHostCommand(msHost.getMsid()); executeCmd(msHost, cmds); - msHostDao.updateState(msHost.getId(), State.PreparingForShutDown); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @@ -375,7 +414,6 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen cmds[0] = new TriggerShutdownManagementServerHostCommand(msHost.getMsid()); executeCmd(msHost, cmds); - msHostDao.updateState(msHost.getId(), State.ShuttingDown); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @@ -395,7 +433,6 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen cmds[0] = new CancelShutdownManagementServerHostCommand(msHost.getMsid()); executeCmd(msHost, cmds); - msHostDao.updateState(msHost.getId(), State.Up); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @@ -426,7 +463,8 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen checkAnyMsInPreparingStates("prepare for maintenance"); - if (indirectAgentLB.haveAgentBasedHosts(msHost.getMsid())) { + boolean ignoreMaintenanceHosts = ManagementServerMaintenanceIgnoreMaintenanceHosts.value(); + if (indirectAgentLB.haveAgentBasedHosts(msHost.getMsid(), ignoreMaintenanceHosts)) { List indirectAgentMsList = indirectAgentLB.getManagementServerList(); indirectAgentMsList.remove(msHost.getServiceIP()); List nonUpMsList = msHostDao.listNonUpStateMsIPs(); @@ -437,10 +475,9 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } final Command[] cmds = new Command[1]; - cmds[0] = new PrepareForMaintenanceManagementServerHostCommand(msHost.getMsid(), cmd.getAlgorithm()); + cmds[0] = new PrepareForMaintenanceManagementServerHostCommand(msHost.getMsid(), cmd.getAlgorithm(), cmd.isForced()); executeCmd(msHost, cmds); - msHostDao.updateState(msHost.getId(), State.PreparingForMaintenance); return prepareMaintenanceResponse(cmd.getManagementServerId()); } @@ -460,7 +497,11 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen cmds[0] = new CancelMaintenanceManagementServerHostCommand(msHost.getMsid()); executeCmd(msHost, cmds); - msHostDao.updateState(msHost.getId(), State.Up); + if (cmd.getRebalance()) { + logger.info("Propagate MS list and rebalance indirect agents"); + indirectAgentLB.propagateMSListToAgents(true); + } + return prepareMaintenanceResponse(cmd.getManagementServerId()); } @@ -485,12 +526,14 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen @Override public void cancelPreparingForMaintenance(ManagementServerHostVO msHost) { - resetPreparingForMaintenance(); - this.preparingForShutdown = false; - this.shutdownTriggered = false; + resetMaintenanceParams(); + resetShutdownParams(); jobManager.enableAsyncJobs(); if (msHost == null) { msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + throw new CloudRuntimeException("Invalid node id for the management server"); + } } onCancelPreparingForMaintenance(); msHostDao.updateState(msHost.getId(), State.Up); @@ -546,17 +589,21 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - ManagementServerMaintenanceTimeoutInMins + ManagementServerMaintenanceTimeoutInMins, ManagementServerMaintenanceIgnoreMaintenanceHosts }; } private final class CheckPendingJobsTask extends ManagedContextRunnable { private ManagementServerMaintenanceManager managementServerMaintenanceManager; + private boolean ignoreMaintenanceHosts = false; private boolean agentsTransferTriggered = false; + private boolean forceMaintenance = false; - public CheckPendingJobsTask(ManagementServerMaintenanceManager managementServerMaintenanceManager) { + public CheckPendingJobsTask(ManagementServerMaintenanceManager managementServerMaintenanceManager, boolean ignoreMaintenanceHosts, boolean forceMaintenance) { this.managementServerMaintenanceManager = managementServerMaintenanceManager; + this.ignoreMaintenanceHosts = ignoreMaintenanceHosts; + this.forceMaintenance = forceMaintenance; } @Override @@ -570,6 +617,19 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } if (managementServerMaintenanceManager.isPreparingForMaintenance() && isMaintenanceWindowExpired()) { + if (forceMaintenance) { + logger.debug("Maintenance window timeout, MS is forced to Maintenance Mode"); + ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + logger.warn("Unable to find the management server, invalid node id"); + return; + } + msHostDao.updateState(msHost.getId(), State.Maintenance); + managementServerMaintenanceManager.onMaintenance(); + managementServerMaintenanceManager.cancelWaitForPendingJobs(); + return; + } + logger.debug("Maintenance window timeout, terminating the pending jobs check timer task"); managementServerMaintenanceManager.cancelPreparingForMaintenance(null); managementServerMaintenanceManager.cancelWaitForPendingJobs(); @@ -577,9 +637,11 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } long totalPendingJobs = managementServerMaintenanceManager.countPendingJobs(ManagementServerNode.getManagementServerId()); - int totalAgents = hostDao.countByMs(ManagementServerNode.getManagementServerId()); - String msg = String.format("Checking for triggered maintenance or shutdown... shutdownTriggered [%b] AllowAsyncJobs [%b] PendingJobCount [%d] AgentsCount [%d]", - managementServerMaintenanceManager.isShutdownTriggered(), managementServerMaintenanceManager.isAsyncJobsEnabled(), totalPendingJobs, totalAgents); + + long totalAgents = totalAgentsInMs(); + + String msg = String.format("Checking for triggered maintenance or shutdown... shutdownTriggered [%b] preparingForShutdown[%b] preparingForMaintenance[%b] AllowAsyncJobs [%b] PendingJobCount [%d] AgentsCount [%d]", + managementServerMaintenanceManager.isShutdownTriggered(), managementServerMaintenanceManager.isPreparingForShutdown(), managementServerMaintenanceManager.isPreparingForMaintenance(), managementServerMaintenanceManager.isAsyncJobsEnabled(), totalPendingJobs, totalAgents); logger.debug(msg); if (totalPendingJobs > 0) { @@ -594,6 +656,10 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen } if (managementServerMaintenanceManager.isPreparingForMaintenance()) { ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + logger.warn("Unable to find the management server, invalid node id"); + return; + } if (totalAgents == 0) { logger.info("MS is in Maintenance Mode"); msHostDao.updateState(msHost.getId(), State.Maintenance); @@ -609,7 +675,7 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen agentsTransferTriggered = true; logger.info(String.format("Preparing for maintenance - migrating agents from management server node %d (id: %s)", ManagementServerNode.getManagementServerId(), msHost.getUuid())); - boolean agentsMigrated = indirectAgentLB.migrateAgents(msHost.getUuid(), ManagementServerNode.getManagementServerId(), managementServerMaintenanceManager.getLbAlgorithm(), remainingMaintenanceWindowInMs()); + boolean agentsMigrated = indirectAgentLB.migrateAgents(msHost.getUuid(), ManagementServerNode.getManagementServerId(), managementServerMaintenanceManager.getLbAlgorithm(), remainingMaintenanceWindowInMs(), ignoreMaintenanceHosts); if (!agentsMigrated) { logger.warn(String.format("Unable to prepare for maintenance, cannot migrate indirect agents on this management server node %d (id: %s)", ManagementServerNode.getManagementServerId(), msHost.getUuid())); managementServerMaintenanceManager.cancelPreparingForMaintenance(msHost); @@ -617,18 +683,20 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen return; } - if(!agentMgr.transferDirectAgentsFromMS(msHost.getUuid(), ManagementServerNode.getManagementServerId(), remainingMaintenanceWindowInMs())) { + if(!agentMgr.transferDirectAgentsFromMS(msHost.getUuid(), ManagementServerNode.getManagementServerId(), remainingMaintenanceWindowInMs(), ignoreMaintenanceHosts)) { logger.warn(String.format("Unable to prepare for maintenance, cannot transfer direct agents on this management server node %d (id: %s)", ManagementServerNode.getManagementServerId(), msHost.getUuid())); managementServerMaintenanceManager.cancelPreparingForMaintenance(msHost); managementServerMaintenanceManager.cancelWaitForPendingJobs(); - return; } } else if (managementServerMaintenanceManager.isPreparingForShutdown()) { logger.info("MS is Ready To Shutdown"); ManagementServerHostVO msHost = msHostDao.findByMsid(ManagementServerNode.getManagementServerId()); + if (msHost == null) { + logger.warn("Unable to find the management server, invalid node id"); + return; + } msHostDao.updateState(msHost.getId(), State.ReadyToShutDown); managementServerMaintenanceManager.cancelWaitForPendingJobs(); - return; } } catch (final Exception e) { logger.error("Error trying to check/run pending jobs task", e); @@ -648,5 +716,14 @@ public class ManagementServerMaintenanceManagerImpl extends ManagerBase implemen long remainingMaintenanceWindowTimeInMs = (ManagementServerMaintenanceTimeoutInMins.value().longValue() * 60 * 1000) - maintenanceElapsedTimeInMs; return (remainingMaintenanceWindowTimeInMs > 0) ? remainingMaintenanceWindowTimeInMs : 0; } + + private long totalAgentsInMs() { + /* Any Host in Maintenance state could block moving Management Server to Maintenance state, exclude those Hosts from total agents count + * To exclude maintenance states use values from ResourceState as source of truth + */ + List statesToExclude = ignoreMaintenanceHosts ? ResourceState.s_maintenanceStates : List.of(); + return hostDao.countHostsByMsResourceStateTypeAndHypervisorType(ManagementServerNode.getManagementServerId(), statesToExclude, + IndirectAgentLBServiceImpl.agentValidHostTypes, null); + } } } diff --git a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/command/PrepareForMaintenanceManagementServerHostCommand.java b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/command/PrepareForMaintenanceManagementServerHostCommand.java index 8f2a4e62b32..ad96454b054 100644 --- a/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/command/PrepareForMaintenanceManagementServerHostCommand.java +++ b/plugins/maintenance/src/main/java/org/apache/cloudstack/maintenance/command/PrepareForMaintenanceManagementServerHostCommand.java @@ -20,17 +20,23 @@ package org.apache.cloudstack.maintenance.command; public class PrepareForMaintenanceManagementServerHostCommand extends BaseShutdownManagementServerHostCommand { String lbAlgorithm; + boolean forced; public PrepareForMaintenanceManagementServerHostCommand(long msId) { super(msId); } - public PrepareForMaintenanceManagementServerHostCommand(long msId, String lbAlgorithm) { + public PrepareForMaintenanceManagementServerHostCommand(long msId, String lbAlgorithm, boolean forced) { super(msId); this.lbAlgorithm = lbAlgorithm; + this.forced = forced; } public String getLbAlgorithm() { return lbAlgorithm; } + + public boolean isForced() { + return forced; + } } diff --git a/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java b/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java index 9fe33aa6c54..280d1eaf9eb 100644 --- a/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java +++ b/plugins/maintenance/src/test/java/org/apache/cloudstack/maintenance/ManagementServerMaintenanceManagerImplTest.java @@ -92,6 +92,8 @@ public class ManagementServerMaintenanceManagerImplTest { @Test public void prepareForShutdown() { Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); spy.prepareForShutdown(); Mockito.verify(jobManagerMock).disableAsyncJobs(); @@ -106,6 +108,9 @@ public class ManagementServerMaintenanceManagerImplTest { @Test public void cancelShutdown() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); Assert.assertThrows(CloudRuntimeException.class, () -> { spy.cancelShutdown(); }); @@ -115,6 +120,8 @@ public class ManagementServerMaintenanceManagerImplTest { public void triggerShutdown() { Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); Mockito.lenient().when(spy.isShutdownTriggered()).thenReturn(false); + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); spy.triggerShutdown(); Mockito.verify(jobManagerMock).disableAsyncJobs(); @@ -305,43 +312,44 @@ public class ManagementServerMaintenanceManagerImplTest { @Test public void prepareForMaintenanceAndCancelFromMaintenanceState() { Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); - spy.prepareForMaintenance("static"); + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); + spy.prepareForMaintenance("static", false); Mockito.verify(jobManagerMock).disableAsyncJobs(); Assert.assertThrows(CloudRuntimeException.class, () -> { - spy.prepareForMaintenance("static"); + spy.prepareForMaintenance("static", false); }); - ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Maintenance); - Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); spy.cancelMaintenance(); Mockito.verify(jobManagerMock).enableAsyncJobs(); - Mockito.verify(spy, Mockito.times(1)).onCancelMaintenance(); } @Test public void prepareForMaintenanceAndCancelFromPreparingForMaintenanceState() { Mockito.doNothing().when(jobManagerMock).disableAsyncJobs(); - spy.prepareForMaintenance("static"); + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); + spy.prepareForMaintenance("static", false); Mockito.verify(jobManagerMock).disableAsyncJobs(); Assert.assertThrows(CloudRuntimeException.class, () -> { - spy.prepareForMaintenance("static"); + spy.prepareForMaintenance("static", false); }); - ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.PreparingForMaintenance); - Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); Mockito.doNothing().when(jobManagerMock).enableAsyncJobs(); spy.cancelMaintenance(); Mockito.verify(jobManagerMock).enableAsyncJobs(); - Mockito.verify(spy, Mockito.times(1)).onCancelPreparingForMaintenance(); } @Test public void cancelMaintenance() { + ManagementServerHostVO msHost = mock(ManagementServerHostVO.class); + Mockito.when(msHost.getState()).thenReturn(ManagementServerHost.State.Up); + Mockito.when(msHostDao.findByMsid(anyLong())).thenReturn(msHost); Assert.assertThrows(CloudRuntimeException.class, () -> { spy.cancelMaintenance(); }); @@ -455,7 +463,7 @@ public class ManagementServerMaintenanceManagerImplTest { Mockito.when(msHostDao.listNonUpStateMsIPs()).thenReturn(new ArrayList<>()); PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); Mockito.when(cmd.getManagementServerId()).thenReturn(1L); - Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(true); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(true); Mockito.when(indirectAgentLBMock.getManagementServerList()).thenReturn(new ArrayList<>()); Assert.assertThrows(CloudRuntimeException.class, () -> { @@ -476,7 +484,7 @@ public class ManagementServerMaintenanceManagerImplTest { Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); Mockito.when(cmd.getManagementServerId()).thenReturn(1L); - Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(false); Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn(null); Assert.assertThrows(CloudRuntimeException.class, () -> { @@ -497,7 +505,7 @@ public class ManagementServerMaintenanceManagerImplTest { Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); Mockito.when(cmd.getManagementServerId()).thenReturn(1L); - Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(false); Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Failed"); Assert.assertThrows(CloudRuntimeException.class, () -> { @@ -518,7 +526,7 @@ public class ManagementServerMaintenanceManagerImplTest { Mockito.when(msHostDao.findById(1L)).thenReturn(msHost1); PrepareForMaintenanceCmd cmd = mock(PrepareForMaintenanceCmd.class); Mockito.when(cmd.getManagementServerId()).thenReturn(1L); - Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong())).thenReturn(false); + Mockito.when(indirectAgentLBMock.haveAgentBasedHosts(anyLong(), anyBoolean())).thenReturn(false); Mockito.when(hostDao.listByMs(anyLong())).thenReturn(new ArrayList<>()); Mockito.when(clusterManagerMock.execute(anyString(), anyLong(), anyString(), anyBoolean())).thenReturn("Success"); diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java index 1ba1cc0fcc3..b7631f78143 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxApiClient.java @@ -474,7 +474,6 @@ public class NsxApiClient { } } - protected void removeSegment(String segmentName, long zoneId) { logger.debug(String.format("Removing the segment with ID %s", segmentName)); Segments segmentService = (Segments) nsxService.apply(Segments.class); diff --git a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java index e1b37a8d653..85d203f4125 100644 --- a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java +++ b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java @@ -94,6 +94,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.NetUtils; import com.cloud.vm.NicProfile; import com.cloud.vm.ReservationContext; import com.cloud.vm.VMInstanceVO; @@ -121,6 +122,7 @@ import javax.naming.ConfigurationException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -861,17 +863,17 @@ public class NsxElement extends AdapterBase implements DhcpServiceProvider, Dns * Replace 0.0.0.0/0 to ANY on each occurrence */ protected List transformCidrListValues(List sourceCidrList) { - List list = new ArrayList<>(); + Set set = new HashSet<>(); if (org.apache.commons.collections.CollectionUtils.isNotEmpty(sourceCidrList)) { for (String cidr : sourceCidrList) { - if (cidr.equals("0.0.0.0/0")) { - list.add("ANY"); + if (cidr.equals(NetUtils.ALL_IP4_CIDRS) || cidr.equals(NetUtils.ALL_IP6_CIDRS)) { + set.add("ANY"); } else { - list.add(cidr); + set.add(cidr); } } } - return list; + return set.stream().sorted().collect(Collectors.toList()); } @Override diff --git a/plugins/storage-allocators/random/src/main/java/org/apache/cloudstack/storage/allocator/RandomStoragePoolAllocator.java b/plugins/storage-allocators/random/src/main/java/org/apache/cloudstack/storage/allocator/RandomStoragePoolAllocator.java index 831e5d2a260..5acab848236 100644 --- a/plugins/storage-allocators/random/src/main/java/org/apache/cloudstack/storage/allocator/RandomStoragePoolAllocator.java +++ b/plugins/storage-allocators/random/src/main/java/org/apache/cloudstack/storage/allocator/RandomStoragePoolAllocator.java @@ -66,7 +66,7 @@ public class RandomStoragePoolAllocator extends AbstractStoragePoolAllocator { StoragePool pol = (StoragePool)this.dataStoreMgr.getPrimaryDataStore(pool.getId()); if (filter(avoid, pol, dskCh, plan)) { - logger.trace(String.format("Found suitable storage pool [%s], adding to list.", pool)); + logger.trace("Found suitable storage pool [{}], adding to list.", pool); suitablePools.add(pol); } } diff --git a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java index 31159e7d3d9..a78164f603d 100644 --- a/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java +++ b/plugins/storage/sharedfs/storagevm/src/main/java/org/apache/cloudstack/storage/sharedfs/lifecycle/StorageVmSharedFSLifeCycle.java @@ -199,7 +199,7 @@ public class StorageVmSharedFSLifeCycle implements SharedFSLifeCycle { customParameterMap, null, null, null, null, true, UserVmManager.SHAREDFSVM, null); vmContext.setEventResourceId(vm.getId()); - userVmService.startVirtualMachine(vm); + userVmService.startVirtualMachine(vm, null); } catch (InsufficientCapacityException ex) { if (vm != null) { expungeVm(vm.getId()); @@ -243,7 +243,7 @@ public class StorageVmSharedFSLifeCycle implements SharedFSLifeCycle { @Override public void startSharedFS(SharedFS sharedFS) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException { UserVmVO vm = userVmDao.findById(sharedFS.getVmId()); - userVmService.startVirtualMachine(vm); + userVmService.startVirtualMachine(vm, null); } @Override diff --git a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java index d68e9cbd110..f6cc3c1e216 100644 --- a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java +++ b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java @@ -17,6 +17,29 @@ package com.cloud.hypervisor.kvm.storage; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.storage.Storage; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Storage.ProvisioningType; +import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.OutputInterpreter; +import com.cloud.utils.script.Script; +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; +import org.apache.cloudstack.storage.datastore.util.StorPoolUtil; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.libvirt.LibvirtException; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -26,19 +49,7 @@ import java.util.Calendar; import java.util.HashMap; import java.util.List; import java.util.Map; - -import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; - -import com.cloud.agent.api.to.DiskTO; -import com.cloud.storage.Storage; -import com.cloud.storage.Storage.ImageFormat; -import com.cloud.storage.Storage.ProvisioningType; -import com.cloud.storage.Storage.StoragePoolType; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.script.OutputInterpreter; -import com.cloud.utils.script.Script; +import java.util.UUID; public class StorPoolStorageAdaptor implements StorageAdaptor { public static void SP_LOG(String fmt, Object... args) { @@ -149,6 +160,10 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { } public static boolean attachOrDetachVolume(String command, String type, String volumeUuid) { + if (volumeUuid == null) { + LOGGER.debug("Could not attach volume. The volume ID is null"); + return false; + } final String name = getVolumeNameFromPath(volumeUuid, true); if (name == null) { return false; @@ -345,11 +360,85 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { throw new UnsupportedOperationException("A folder cannot be created in this configuration."); } - public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, - KVMStoragePool destPool, ImageFormat format, int timeout) { + @Override + public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, + PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { return null; } + @Override + public KVMPhysicalDisk createTemplateFromDirectDownloadFile(String templateFilePath, String destTemplatePath, + KVMStoragePool destPool, ImageFormat format, int timeout) { + if (StringUtils.isEmpty(templateFilePath) || destPool == null) { + throw new CloudRuntimeException( + "Unable to create template from direct download template file due to insufficient data"); + } + + File sourceFile = new File(templateFilePath); + if (!sourceFile.exists()) { + throw new CloudRuntimeException( + "Direct download template file " + templateFilePath + " does not exist on this host"); + } + + if (!StoragePoolType.StorPool.equals(destPool.getType())) { + throw new CloudRuntimeException("Unsupported storage pool type: " + destPool.getType().toString()); + } + + if (!Storage.ImageFormat.QCOW2.equals(format)) { + throw new CloudRuntimeException("Unsupported template format: " + format.toString()); + } + + String srcTemplateFilePath = templateFilePath; + KVMPhysicalDisk destDisk = null; + QemuImgFile srcFile = null; + QemuImgFile destFile = null; + String templateName = UUID.randomUUID().toString(); + String volume = null; + try { + + srcTemplateFilePath = extractTemplate(templateFilePath, sourceFile, srcTemplateFilePath, templateName); + + QemuImg.PhysicalDiskFormat srcFileFormat = QemuImg.PhysicalDiskFormat.QCOW2; + + srcFile = new QemuImgFile(srcTemplateFilePath, srcFileFormat); + + String spTemplate = destPool.getUuid().split(";")[0]; + + QemuImg qemu = new QemuImg(timeout); + OutputInterpreter.AllLinesParser parser = createStorPoolVolume(destPool, srcFile, qemu, spTemplate); + + String response = parser.getLines(); + + LOGGER.debug(response); + volume = StorPoolUtil.devPath(getNameFromResponse(response, false, false)); + attachOrDetachVolume("attach", "volume", volume); + destDisk = destPool.getPhysicalDisk(volume); + if (destDisk == null) { + throw new CloudRuntimeException( + "Failed to find the disk: " + volume + " of the storage pool: " + destPool.getUuid()); + } + + destFile = new QemuImgFile(destDisk.getPath(), QemuImg.PhysicalDiskFormat.RAW); + + qemu.convert(srcFile, destFile); + parser = volumeSnapshot(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true), spTemplate); + response = parser.getLines(); + LOGGER.debug(response); + String newPath = StorPoolUtil.devPath(getNameFromResponse(response, false, true)); + destDisk = destPool.getPhysicalDisk(newPath); + } catch (QemuImgException | LibvirtException e) { + destDisk = null; + } finally { + if (volume != null) { + attachOrDetachVolume("detach", "volume", volume); + volumeDelete(StorPoolStorageAdaptor.getVolumeNameFromPath(volume, true)); + } + Script.runSimpleBashScript("rm -f " + srcTemplateFilePath); + } + + return destDisk; + } + @Override public boolean createFolder(String uuid, String path, String localPath) { return false; @@ -367,9 +456,104 @@ public class StorPoolStorageAdaptor implements StorageAdaptor { return null; } - @Override - public KVMPhysicalDisk createDiskFromTemplateBacking(KVMPhysicalDisk template, String name, - PhysicalDiskFormat format, long size, KVMStoragePool destPool, int timeout, byte[] passphrase) { - return null; + private OutputInterpreter.AllLinesParser createStorPoolVolume(KVMStoragePool destPool, QemuImgFile srcFile, + QemuImg qemu, String templateUuid) throws QemuImgException, LibvirtException { + Map info = qemu.info(srcFile); + Map reqParams = new HashMap<>(); + reqParams.put("template", templateUuid); + reqParams.put("size", info.get("virtual_size")); + Map tags = new HashMap<>(); + tags.put("cs", "template"); + reqParams.put("tags", tags); + Gson gson = new Gson(); + String js = gson.toJson(reqParams); + + Script sc = createStorPoolRequest(js, "VolumeCreate", null,true); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + String res = sc.execute(parser); + if (res != null) { + throw new CloudRuntimeException("Could not create volume due to: " + res); + } + return parser; + } + + private OutputInterpreter.AllLinesParser volumeSnapshot(String volumeName, String templateUuid) { + Map reqParams = new HashMap<>(); + reqParams.put("template", templateUuid); + Gson gson = new Gson(); + String js = gson.toJson(reqParams); + + Script sc = createStorPoolRequest(js, "VolumeSnapshot", volumeName,true); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + String res = sc.execute(parser); + if (res != null) { + throw new CloudRuntimeException("Could not snapshot volume due to: " + res); + } + return parser; + } + + private OutputInterpreter.AllLinesParser volumeDelete(String volumeName) { + Script sc = createStorPoolRequest(null, "VolumeDelete", volumeName, false); + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + + String res = sc.execute(parser); + if (res != null) { + throw new CloudRuntimeException("Could not delete volume due to: " + res); + } + return parser; + } + @NotNull + private static Script createStorPoolRequest(String js, String apiCall, String param, boolean jsonRequired) { + Script sc = new Script("storpool_req", 0, LOGGER); + sc.add("-P"); + sc.add("-M"); + if (jsonRequired) { + sc.add("--json"); + sc.add(js); + } + sc.add(apiCall); + if (param != null) { + sc.add(param); + } + return sc; + } + + private String extractTemplate(String templateFilePath, File sourceFile, String srcTemplateFilePath, + String templateName) { + if (isTemplateExtractable(templateFilePath)) { + srcTemplateFilePath = sourceFile.getParent() + "/" + templateName; + String extractCommand = getExtractCommandForDownloadedFile(templateFilePath, srcTemplateFilePath); + Script.runSimpleBashScript(extractCommand); + Script.runSimpleBashScript("rm -f " + templateFilePath); + } + return srcTemplateFilePath; + } + + private boolean isTemplateExtractable(String templatePath) { + String type = Script.runSimpleBashScript("file " + templatePath + " | awk -F' ' '{print $2}'"); + return type.equalsIgnoreCase("bzip2") || type.equalsIgnoreCase("gzip") || type.equalsIgnoreCase("zip"); + } + + private String getExtractCommandForDownloadedFile(String downloadedTemplateFile, String templateFile) { + if (downloadedTemplateFile.endsWith(".zip")) { + return "unzip -p " + downloadedTemplateFile + " | cat > " + templateFile; + } else if (downloadedTemplateFile.endsWith(".bz2")) { + return "bunzip2 -c " + downloadedTemplateFile + " > " + templateFile; + } else if (downloadedTemplateFile.endsWith(".gz")) { + return "gunzip -c " + downloadedTemplateFile + " > " + templateFile; + } else { + throw new CloudRuntimeException("Unable to extract template " + downloadedTemplateFile); + } + } + + private String getNameFromResponse(String resp, boolean tildeNeeded, boolean isSnapshot) { + JsonParser jsonParser = new JsonParser(); + JsonObject respObj = (JsonObject) jsonParser.parse(resp); + JsonPrimitive data = isSnapshot ? respObj.getAsJsonPrimitive("snapshotGlobalId") : respObj.getAsJsonPrimitive("globalId"); + String name = data !=null ? data.getAsString() : null; + name = name != null ? name.startsWith("~") && !tildeNeeded ? name.split("~")[1] : name : name; + return name; } } diff --git a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStoragePool.java b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStoragePool.java index ac7138420c0..ab5dc03d343 100644 --- a/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStoragePool.java +++ b/plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStoragePool.java @@ -29,6 +29,7 @@ import com.cloud.agent.api.to.HostTO; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef; import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.script.OutputInterpreter; @@ -302,4 +303,11 @@ public class StorPoolStoragePool implements KVMStoragePool { public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUuidListString, String vmActivityCheckPath, long duration) { return checkingHeartBeat(pool, host); } + + @Override + public void customizeLibvirtDiskDef(LibvirtVMDef.DiskDef disk) { + if (LibvirtVMDef.DiskDef.DiskBus.VIRTIO.equals(disk.getBusType()) || LibvirtVMDef.DiskDef.DiskBus.SCSI.equals(disk.getBusType())) { + disk.setDiscard(LibvirtVMDef.DiskDef.DiscardType.UNMAP); + } + } } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java index 6ca67cb5923..d8aa9d48e7d 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java @@ -222,7 +222,6 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { StorPoolUtil.volumeRemoveTags(StorPoolStorageAdaptor.getVolumeNameFromPath(volume.getPath(), true), conn); } } - } private void updateStoragePool(final long poolId, final long deltaUsedBytes) { @@ -327,19 +326,14 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { Long vmId, SpConnectionDesc conn) { SpApiResponse resp = new SpApiResponse(); Map tags = StorPoolHelper.addStorPoolTags(name, getVMInstanceUUID(vmId), "volume", getVcPolicyTag(vmId), tier); - if (tier != null || template != null) { - StorPoolUtil.spLog( - "Creating volume [%s] with template [%s] or tier tags [%s] described in disk/service offerings details", - vinfo.getUuid(), template, tier); - resp = StorPoolUtil.volumeCreate(size, null, template, tags, conn); - } else { - StorPoolUtil.spLog( - "StorpoolPrimaryDataStoreDriver.createAsync volume: name=%s, uuid=%s, isAttached=%s vm=%s, payload=%s, template: %s", - vinfo.getName(), vinfo.getUuid(), vinfo.isAttachedVM(), vinfo.getAttachedVmName(), - vinfo.getpayload(), conn.getTemplateName()); - resp = StorPoolUtil.volumeCreate(name, null, size, getVMInstanceUUID(vinfo.getInstanceId()), null, - "volume", vinfo.getMaxIops(), conn); + if (vinfo.getDeviceId() != null) { + tags.put("disk", vinfo.getDeviceId().toString()); } + if (template == null) { + template = conn.getTemplateName(); + } + StorPoolVolumeDef volume = new StorPoolVolumeDef(null, size, tags, null, vinfo.getMaxIops(), template, null, null, null); + resp = StorPoolUtil.volumeCreate(volume, conn); return resp; } @@ -827,20 +821,24 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { if (tier == null) { template = getTemplateFromOfferingDetail(vinfo.getDiskOfferingId()); } - } - - if (tier != null || template != null) { - Map tags = StorPoolHelper.addStorPoolTags(name, getVMInstanceUUID(vmId), "volume", getVcPolicyTag(vmId), tier); - StorPoolUtil.spLog( "Creating volume [%s] with template [%s] or tier tags [%s] described in disk/service offerings details", vinfo.getUuid(), template, tier); - resp = StorPoolUtil.volumeCreate(size, parentName, template, tags, conn); - } else { - resp = StorPoolUtil.volumeCreate(name, parentName, size, getVMInstanceUUID(vmId), - getVcPolicyTag(vmId), "volume", vinfo.getMaxIops(), conn); } + Map tags = StorPoolHelper.addStorPoolTags(name, getVMInstanceUUID(vmId), "volume", getVcPolicyTag(vmId), tier); + + if (vinfo.getDeviceId() != null) { + tags.put("disk", vinfo.getDeviceId().toString()); + } + + if (template == null) { + template = conn.getTemplateName(); + } + + StorPoolVolumeDef volumeDef = new StorPoolVolumeDef(null, size, tags, parentName, null, template, null, null, null); + resp = StorPoolUtil.volumeCreate(volumeDef, conn); + if (resp.getError() == null) { updateStoragePool(dstData.getDataStore().getId(), vinfo.getSize()); updateVolumePoolType(vinfo); @@ -1309,7 +1307,13 @@ public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver { SpConnectionDesc conn = StorPoolUtil.getSpConnection(poolVO.getUuid(), poolVO.getId(), storagePoolDetailsDao, primaryStoreDao); String volName = StorPoolStorageAdaptor.getVolumeNameFromPath(volume.getPath(), true); VMInstanceVO userVM = vmInstanceDao.findById(vmId); - SpApiResponse resp = StorPoolUtil.volumeUpdateIopsAndTags(volName, volume.getInstanceId() != null ? userVM.getUuid() : "", null, conn, getVcPolicyTag(vmId)); + Map tags = StorPoolHelper.addStorPoolTags(null, userVM.getUuid(), null, getVcPolicyTag(vmId), null); + if (volume.getDeviceId() != null) { + tags.put("disk", volume.getDeviceId().toString()); + } + StorPoolVolumeDef spVolume = new StorPoolVolumeDef(volName, null, tags, null, null, null, null, null, null); + + SpApiResponse resp = StorPoolUtil.volumeUpdate(spVolume, conn); if (resp.getError() != null) { logger.warn(String.format("Could not update VC policy tags of a volume with id [%s]", volume.getUuid())); } diff --git a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java index fd62157f136..fa9248033bf 100644 --- a/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java +++ b/plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java @@ -520,6 +520,10 @@ public class StorPoolUtil { return POST("MultiCluster/VolumeCreate", json, conn); } + public static SpApiResponse volumeCreate(StorPoolVolumeDef volume, SpConnectionDesc conn) { + return POST("MultiCluster/VolumeCreate", volume, conn); + } + public static SpApiResponse volumeCreate(SpConnectionDesc conn) { Map json = new LinkedHashMap<>(); json.put("name", ""); @@ -568,6 +572,7 @@ public class StorPoolUtil { public static SpApiResponse volumeRemoveTags(String name, SpConnectionDesc conn) { Map json = new HashMap<>(); Map tags = StorPoolHelper.addStorPoolTags(null, "", null, "", null); + tags.put("disk", ""); json.put("tags", tags); return POST("MultiCluster/VolumeUpdate/" + name, json, conn); } diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java index e6f23ef8ab3..1c759c6a45e 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapContextFactory.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.ldap; +import java.io.FileInputStream; import java.io.IOException; import java.util.Hashtable; @@ -24,6 +25,7 @@ import javax.naming.Context; import javax.naming.NamingException; import javax.naming.ldap.InitialLdapContext; import javax.naming.ldap.LdapContext; +import java.security.KeyStore; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; @@ -52,14 +54,14 @@ public class LdapContextFactory { return createInitialDirContext(bindPrincipal, bindPassword, providerUrl, true, domainId); } - private LdapContext createInitialDirContext(final String principal, final String password, final boolean isSystemContext, Long domainId) throws NamingException, IOException { + private LdapContext createInitialDirContext(final String principal, final String password, final boolean isSystemContext, Long domainId) throws NamingException { return createInitialDirContext(principal, password, null, isSystemContext, domainId); } private LdapContext createInitialDirContext(final String principal, final String password, final String providerUrl, final boolean isSystemContext, Long domainId) - throws NamingException, IOException { + throws NamingException { Hashtable environment = getEnvironment(principal, password, providerUrl, isSystemContext, domainId); - logger.debug("initializing ldap with provider url: " + environment.get(Context.PROVIDER_URL)); + logger.debug("initializing ldap with provider url: {}", environment.get(Context.PROVIDER_URL)); return new InitialLdapContext(environment, null); } @@ -73,8 +75,36 @@ public class LdapContextFactory { if (sslStatus) { logger.info("LDAP SSL enabled."); environment.put(Context.SECURITY_PROTOCOL, "ssl"); - System.setProperty("javax.net.ssl.trustStore", _ldapConfiguration.getTrustStore(domainId)); - System.setProperty("javax.net.ssl.trustStorePassword", _ldapConfiguration.getTrustStorePassword(domainId)); + String trustStore = _ldapConfiguration.getTrustStore(domainId); + String trustStorePassword = _ldapConfiguration.getTrustStorePassword(domainId); + + if (!validateTrustStore(trustStore, trustStorePassword)) { + throw new RuntimeException("Invalid truststore or truststore password"); + } + + System.setProperty("javax.net.ssl.trustStore", trustStore); + System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword); + } + } + + private boolean validateTrustStore(String trustStore, String trustStorePassword) { + if (trustStore == null) { + return true; + } + + if (trustStorePassword == null) { + return false; + } + + try { + KeyStore.getInstance("JKS").load( + new FileInputStream(trustStore), + trustStorePassword.toCharArray() + ); + return true; + } catch (Exception e) { + logger.warn("Failed to validate truststore: {}", e.getMessage()); + return false; } } diff --git a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java index 16914e792a6..2394c0b3d1e 100644 --- a/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java +++ b/plugins/user-authenticators/ldap/src/main/java/org/apache/cloudstack/ldap/LdapManagerImpl.java @@ -184,6 +184,11 @@ public class LdapManagerImpl extends ComponentLifecycleBase implements LdapManag } catch (NamingException | IOException e) { logger.debug("NamingException while doing an LDAP bind", e); throw new InvalidParameterValueException("Unable to bind to the given LDAP server"); + } catch (RuntimeException e) { + if (e.getMessage().contains("Invalid truststore")) { + throw new InvalidParameterValueException("Invalid truststore or truststore password"); + } + throw e; } finally { closeContext(context); } diff --git a/pom.xml b/pom.xml index ae1cb05a02c..81e297608bf 100644 --- a/pom.xml +++ b/pom.xml @@ -117,8 +117,6 @@ 5.9.1 18.0 5.16.1 - 1.0-20081010.060147 - 1.0.1 7.1.0 2.27.2 2.12.2 diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index e4364054d2a..535d4b36479 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -19,8 +19,8 @@ set -e if [ $# -lt 6 ]; then - echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME" - echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4" + echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME [ETCD_VERSION]" + echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4 3.5.1" exit 1 fi @@ -148,6 +148,14 @@ chmod ${kubeadm_file_permissions} "${working_dir}/k8s/kubeadm" echo "Updating imagePullPolicy to IfNotPresent in yaml files..." sed -i "s/imagePullPolicy:.*/imagePullPolicy: IfNotPresent/g" ${working_dir}/*.yaml +if [ -n "${8}" ]; then + # Install etcd dependencies + etcd_dir="${working_dir}/etcd" + mkdir -p "${etcd_dir}" + ETCD_VERSION=v${8} + wget -q --show-progress "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" -O ${etcd_dir}/etcd-linux-amd64.tar.gz +fi + mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}" rm -rf "${iso_dir}" diff --git a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java index f35a0664a85..377b2134d78 100644 --- a/server/src/main/java/com/cloud/alert/AlertManagerImpl.java +++ b/server/src/main/java/com/cloud/alert/AlertManagerImpl.java @@ -89,6 +89,10 @@ import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.concurrency.NamedThreadFactory; import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionStatus; + import org.jetbrains.annotations.Nullable; public class AlertManagerImpl extends ManagerBase implements AlertManager, Configurable { @@ -290,8 +294,13 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size()))); for (Long hostId : hostIds) { futures.put(hostId, executorService.submit(() -> { - final HostVO host = hostDao.findById(hostId); - _capacityMgr.updateCapacityForHost(host); + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final HostVO host = hostDao.findById(hostId); + _capacityMgr.updateCapacityForHost(host); + } + }); return null; })); } @@ -316,13 +325,18 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size()))); for (Long poolId: storagePoolIds) { futures.put(poolId, executorService.submit(() -> { - final StoragePoolVO pool = _storagePoolDao.findById(poolId); - long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); - if (pool.isShared()) { - _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); - } else { - _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); - } + Transaction.execute(new TransactionCallbackNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) { + final StoragePoolVO pool = _storagePoolDao.findById(poolId); + long disk = _capacityMgr.getAllocatedPoolCapacity(pool, null); + if (pool.isShared()) { + _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_STORAGE_ALLOCATED, disk); + } else { + _storageMgr.createCapacityEntry(pool, Capacity.CAPACITY_TYPE_LOCAL_STORAGE, disk); + } + } + }); return null; })); } diff --git a/server/src/main/java/com/cloud/api/ApiResponseHelper.java b/server/src/main/java/com/cloud/api/ApiResponseHelper.java index 652480259c9..ef65c4c3120 100644 --- a/server/src/main/java/com/cloud/api/ApiResponseHelper.java +++ b/server/src/main/java/com/cloud/api/ApiResponseHelper.java @@ -39,16 +39,6 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.bgp.ASNumber; -import com.cloud.bgp.ASNumberRange; -import com.cloud.dc.ASNumberRangeVO; -import com.cloud.dc.ASNumberVO; -import com.cloud.dc.VlanDetailsVO; -import com.cloud.dc.dao.ASNumberDao; -import com.cloud.dc.dao.ASNumberRangeDao; -import com.cloud.dc.dao.VlanDetailsDao; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.BucketVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.affinity.AffinityGroup; @@ -63,12 +53,12 @@ import org.apache.cloudstack.api.BaseResponseWithAssociatedNetwork; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd; +import org.apache.cloudstack.api.response.ASNRangeResponse; +import org.apache.cloudstack.api.response.ASNumberResponse; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.ApplicationLoadBalancerInstanceResponse; import org.apache.cloudstack.api.response.ApplicationLoadBalancerResponse; import org.apache.cloudstack.api.response.ApplicationLoadBalancerRuleResponse; -import org.apache.cloudstack.api.response.ASNRangeResponse; -import org.apache.cloudstack.api.response.ASNumberResponse; import org.apache.cloudstack.api.response.AsyncJobResponse; import org.apache.cloudstack.api.response.AutoScalePolicyResponse; import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse; @@ -99,10 +89,10 @@ import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.DomainRouterResponse; import org.apache.cloudstack.api.response.EventResponse; import org.apache.cloudstack.api.response.ExtractResponse; -import org.apache.cloudstack.api.response.SharedFSResponse; import org.apache.cloudstack.api.response.FirewallResponse; import org.apache.cloudstack.api.response.FirewallRuleResponse; import org.apache.cloudstack.api.response.GlobalLoadBalancerResponse; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.api.response.GuestOsMappingResponse; import org.apache.cloudstack.api.response.GuestVlanRangeResponse; @@ -164,6 +154,7 @@ import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.SecurityGroupRuleResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.ServiceResponse; +import org.apache.cloudstack.api.response.SharedFSResponse; import org.apache.cloudstack.api.response.Site2SiteCustomerGatewayResponse; import org.apache.cloudstack.api.response.Site2SiteVpnConnectionResponse; import org.apache.cloudstack.api.response.Site2SiteVpnGatewayResponse; @@ -230,10 +221,10 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.storage.sharedfs.SharedFS; -import org.apache.cloudstack.storage.sharedfs.query.vo.SharedFSJoinVO; import org.apache.cloudstack.storage.object.Bucket; import org.apache.cloudstack.storage.object.ObjectStore; +import org.apache.cloudstack.storage.sharedfs.SharedFS; +import org.apache.cloudstack.storage.sharedfs.query.vo.SharedFSJoinVO; import org.apache.cloudstack.usage.Usage; import org.apache.cloudstack.usage.UsageService; import org.apache.cloudstack.usage.UsageTypes; @@ -241,8 +232,8 @@ import org.apache.cloudstack.vm.UnmanagedInstanceTO; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.cloud.agent.api.VgpuTypesInfo; import com.cloud.api.query.ViewResponseHelper; @@ -271,6 +262,8 @@ import com.cloud.api.query.vo.UserVmJoinVO; import com.cloud.api.query.vo.VolumeJoinVO; import com.cloud.api.query.vo.VpcOfferingJoinVO; import com.cloud.api.response.ApiResponseSerializer; +import com.cloud.bgp.ASNumber; +import com.cloud.bgp.ASNumberRange; import com.cloud.capacity.Capacity; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDaoImpl.SummedCapacity; @@ -279,6 +272,8 @@ import com.cloud.configuration.Resource.ResourceOwnerType; import com.cloud.configuration.Resource.ResourceType; import com.cloud.configuration.ResourceCount; import com.cloud.configuration.ResourceLimit; +import com.cloud.dc.ASNumberRangeVO; +import com.cloud.dc.ASNumberVO; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterVO; import com.cloud.dc.DataCenter; @@ -289,7 +284,11 @@ import com.cloud.dc.Pod; import com.cloud.dc.StorageNetworkIpRange; import com.cloud.dc.Vlan; import com.cloud.dc.Vlan.VlanType; +import com.cloud.dc.VlanDetailsVO; import com.cloud.dc.VlanVO; +import com.cloud.dc.dao.ASNumberDao; +import com.cloud.dc.dao.ASNumberRangeDao; +import com.cloud.dc.dao.VlanDetailsDao; import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.event.Event; @@ -299,6 +298,7 @@ import com.cloud.gpu.GPU; import com.cloud.host.ControlState; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.HypervisorCapabilities; import com.cloud.network.GuestVlan; import com.cloud.network.GuestVlanRange; @@ -377,10 +377,13 @@ import com.cloud.projects.ProjectAccount; import com.cloud.projects.ProjectInvitation; import com.cloud.region.ha.GlobalLoadBalancerRule; import com.cloud.resource.RollingMaintenanceManager; +import com.cloud.resource.icon.ResourceIconVO; import com.cloud.server.ResourceIcon; +import com.cloud.server.ResourceIconManager; import com.cloud.server.ResourceTag; import com.cloud.server.ResourceTag.ResourceObjectType; import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.BucketVO; import com.cloud.storage.DataStoreRole; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.GuestOS; @@ -521,6 +524,8 @@ public class ApiResponseHelper implements ResponseGenerator { BgpPeerDao bgpPeerDao; @Inject RoutedIpv4Manager routedIpv4Manager; + @Inject + ResourceIconManager resourceIconManager; @Override public UserResponse createUserResponse(User user) { @@ -3902,6 +3907,30 @@ public class ApiResponseHelper implements ResponseGenerator { return response; } + @Override + public GuestOSCategoryResponse createGuestOSCategoryResponse(GuestOsCategory guestOsCategory) { + return createGuestOSCategoryResponse(guestOsCategory, true); + } + + @Override + public GuestOSCategoryResponse createGuestOSCategoryResponse(GuestOsCategory guestOsCategory, boolean showIcon) { + GuestOSCategoryResponse categoryResponse = new GuestOSCategoryResponse(); + categoryResponse.setId(guestOsCategory.getUuid()); + categoryResponse.setName(guestOsCategory.getName()); + categoryResponse.setFeatured(guestOsCategory.isFeatured()); + categoryResponse.setCreated(guestOsCategory.getCreated()); + if (showIcon) { + ResourceIconVO resourceIcon = ApiDBUtils.getResourceIconByResourceUUID(guestOsCategory.getUuid(), + ResourceObjectType.GuestOsCategory); + if (resourceIcon != null) { + ResourceIconResponse iconResponse = ApiDBUtils.newResourceIconResponse(resourceIcon); + categoryResponse.setResourceIconResponse(iconResponse); + } + } + categoryResponse.setObjectName("oscategory"); + return categoryResponse; + } + @Override public GuestOSResponse createGuestOSResponse(GuestOS guestOS) { GuestOSResponse response = new GuestOSResponse(); @@ -5439,9 +5468,13 @@ public class ApiResponseHelper implements ResponseGenerator { response.setZoneName(zone.getName()); response.setAsNumber(asn.getAsNumber()); ASNumberRangeVO range = asNumberRangeDao.findById(asn.getAsNumberRangeId()); - response.setAsNumberRangeId(range.getUuid()); - String rangeText = String.format("%s-%s", range.getStartASNumber(), range.getEndASNumber()); - response.setAsNumberRange(rangeText); + if (Objects.nonNull(range)) { + response.setAsNumberRangeId(range.getUuid()); + String rangeText = String.format("%s-%s", range.getStartASNumber(), range.getEndASNumber()); + response.setAsNumberRange(rangeText); + } else { + logger.info("Range is null for AS number: "+ asn.getAsNumber()); + } response.setAllocated(asn.getAllocatedTime()); response.setAllocationState(asn.isAllocated() ? "Allocated" : "Free"); if (asn.getVpcId() != null) { @@ -5482,4 +5515,39 @@ public class ApiResponseHelper implements ResponseGenerator { SharedFSJoinVO sharedFSView = ApiDBUtils.newSharedFSView(sharedFS); return ApiDBUtils.newSharedFSResponse(view, sharedFSView); } + + protected Map getResourceIconsUsingOsCategory(List responses) { + Set guestOsCategoryIds = responses.stream().map(TemplateResponse::getOsTypeCategoryId).collect(Collectors.toSet()); + Map guestOsCategoryIcons = + resourceIconManager.getByResourceTypeAndIds(ResourceTag.ResourceObjectType.GuestOsCategory, + guestOsCategoryIds); + Map vmIcons = new HashMap<>(); + for (TemplateResponse response : responses) { + vmIcons.put(response.getId(), guestOsCategoryIcons.get(response.getOsTypeCategoryId())); + } + return vmIcons; + } + + @Override + public void updateTemplateIsoResponsesForIcons(List responses, + ResourceTag.ResourceObjectType type) { + if (CollectionUtils.isEmpty(responses)) { + return; + } + Set uuids = responses.stream().map(TemplateResponse::getId).collect(Collectors.toSet()); + Map templateIcons = resourceIconManager.getByResourceTypeAndUuids(type, uuids); + List noTemplateIconResponses = responses + .stream() + .filter(r -> !templateIcons.containsKey(r.getId())) + .collect(Collectors.toList()); + templateIcons.putAll(getResourceIconsUsingOsCategory(noTemplateIconResponses)); + for (TemplateResponse response : responses) { + ResourceIcon icon = templateIcons.get(response.getId()); + if (icon == null) { + continue; + } + ResourceIconResponse iconResponse = createResourceIconResponse(icon); + response.setResourceIconResponse(iconResponse); + } + } } diff --git a/server/src/main/java/com/cloud/api/ApiServer.java b/server/src/main/java/com/cloud/api/ApiServer.java index b8227ef9d58..34752000907 100644 --- a/server/src/main/java/com/cloud/api/ApiServer.java +++ b/server/src/main/java/com/cloud/api/ApiServer.java @@ -201,6 +201,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer private static final String SANITIZATION_REGEX = "[\n\r]"; private static boolean encodeApiResponse = false; + private boolean isPostRequestsAndTimestampsEnforced = false; /** * Non-printable ASCII characters - numbers 0 to 31 and 127 decimal @@ -284,6 +285,13 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer , "Session cookie is marked as secure if this is enabled. Secure cookies only work when HTTPS is used." , false , ConfigKey.Scope.Global); + static final ConfigKey EnforcePostRequestsAndTimestamps = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED + , Boolean.class + , "enforce.post.requests.and.timestamps" + , "false" + , "Enable/Disable whether the ApiServer should only accept POST requests for state-changing APIs and requests with timestamps." + , false + , ConfigKey.Scope.Global); private static final ConfigKey JSONDefaultContentType = new ConfigKey<> (ConfigKey.CATEGORY_ADVANCED , String.class , "json.content.type" @@ -441,6 +449,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer public boolean start() { Security.addProvider(new BouncyCastleProvider()); Integer apiPort = IntegrationAPIPort.value(); // api port, null by default + isPostRequestsAndTimestampsEnforced = EnforcePostRequestsAndTimestamps.value(); final Long snapshotLimit = ConcurrentSnapshotsThresholdPerHost.value(); if (snapshotLimit == null || snapshotLimit <= 0) { @@ -720,6 +729,11 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer return response; } + @Override + public boolean isPostRequestsAndTimestampsEnforced() { + return isPostRequestsAndTimestampsEnforced; + } + private String getBaseAsyncResponse(final long jobId, final BaseAsyncCmd cmd) { final AsyncJobResponse response = new AsyncJobResponse(); @@ -967,7 +981,6 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer // put the name in a list that we'll sort later final List parameterNames = new ArrayList<>(requestParameters.keySet()); - Collections.sort(parameterNames); String signatureVersion = null; @@ -1019,12 +1032,22 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer } final Date now = new Date(System.currentTimeMillis()); + final Date thresholdTime = new Date(now.getTime() + 15 * 60 * 1000); if (expiresTS.before(now)) { signature = signature.replaceAll(SANITIZATION_REGEX, "_"); apiKey = apiKey.replaceAll(SANITIZATION_REGEX, "_"); logger.debug("Request expired -- ignoring ...sig [{}], apiKey [{}].", signature, apiKey); return false; + } else if (isPostRequestsAndTimestampsEnforced && expiresTS.after(thresholdTime)) { + signature = signature.replaceAll(SANITIZATION_REGEX, "_"); + apiKey = apiKey.replaceAll(SANITIZATION_REGEX, "_"); + logger.debug(String.format("Expiration parameter is set for too long -- ignoring ...sig [%s], apiKey [%s].", signature, apiKey)); + return false; } + } else if (isPostRequestsAndTimestampsEnforced) { + // Force expiration parameter + logger.debug("Signature Version must be 3, and should be along with the Expires parameter -- ignoring request."); + return false; } final TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.CLOUD_DB); @@ -1648,6 +1671,7 @@ public class ApiServer extends ManagerBase implements HttpRequestHandler, ApiSer @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[] { + EnforcePostRequestsAndTimestamps, IntegrationAPIPort, ConcurrentSnapshotsThresholdPerHost, EncodeApiResponse, diff --git a/server/src/main/java/com/cloud/api/ApiServlet.java b/server/src/main/java/com/cloud/api/ApiServlet.java index 4994c42bb4d..41229670c38 100644 --- a/server/src/main/java/com/cloud/api/ApiServlet.java +++ b/server/src/main/java/com/cloud/api/ApiServlet.java @@ -22,8 +22,11 @@ import java.net.URLDecoder; import java.net.UnknownHostException; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; +import java.util.Set; import javax.inject.Inject; import javax.servlet.ServletConfig; @@ -46,6 +49,7 @@ import org.apache.cloudstack.api.command.user.consoleproxy.CreateConsoleEndpoint import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.cloudstack.utils.consoleproxy.ConsoleAccessUtils; +import org.apache.commons.collections.MapUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -78,6 +82,39 @@ public class ApiServlet extends HttpServlet { private static final Logger ACCESSLOGGER = LogManager.getLogger("apiserver." + ApiServlet.class.getName()); private static final String REPLACEMENT = "_"; private static final String LOGGER_REPLACEMENTS = "[\n\r\t]"; + private static final Pattern GET_REQUEST_COMMANDS = Pattern.compile("^(get|list|query|find)(\\w+)+$"); + private static final HashSet GET_REQUEST_COMMANDS_LIST = new HashSet<>(Set.of("isaccountallowedtocreateofferingswithtags", + "readyforshutdown", "cloudianisenabled", "quotabalance", "quotasummary", "quotatarifflist", "quotaisenabled", "quotastatement", "verifyoauthcodeandgetuser")); + private static final HashSet POST_REQUESTS_TO_DISABLE_LOGGING = new HashSet<>(Set.of( + "login", + "oauthlogin", + "createaccount", + "createuser", + "updateuser", + "forgotpassword", + "resetpassword", + "importrole", + "updaterolepermission", + "updateprojectrolepermission", + "createstoragepool", + "addhost", + "updatehostpassword", + "addcluster", + "addvmwaredc", + "configureoutofbandmanagement", + "uploadcustomcertificate", + "addciscovnmcresource", + "addnetscalerloadbalancer", + "createtungstenfabricprovider", + "addnsxcontroller", + "configtungstenfabricservice", + "createnetworkacl", + "updatenetworkaclitem", + "quotavalidateactivationrule", + "quotatariffupdate", + "listandswitchsamlaccount", + "uploadresourceicon" + )); @Inject ApiServerService apiServer; @@ -193,11 +230,24 @@ public class ApiServlet extends HttpServlet { utf8Fixup(req, params); + final Object[] commandObj = params.get(ApiConstants.COMMAND); + final String command = commandObj == null ? null : (String) commandObj[0]; + // logging the request start and end in management log for easy debugging String reqStr = ""; String cleanQueryString = StringUtils.cleanString(req.getQueryString()); if (LOGGER.isDebugEnabled()) { reqStr = auditTrailSb.toString() + " " + cleanQueryString; + if (req.getMethod().equalsIgnoreCase("POST") && org.apache.commons.lang3.StringUtils.isNotBlank(command)) { + if (!POST_REQUESTS_TO_DISABLE_LOGGING.contains(command.toLowerCase()) && !reqParams.containsKey(ApiConstants.USER_DATA)) { + String cleanParamsString = getCleanParamsString(reqParams); + if (org.apache.commons.lang3.StringUtils.isNotBlank(cleanParamsString)) { + reqStr += "\n" + cleanParamsString; + } + } else { + reqStr += " " + command; + } + } LOGGER.debug("===START=== " + reqStr); } @@ -213,8 +263,6 @@ public class ApiServlet extends HttpServlet { responseType = (String)responseTypeParam[0]; } - final Object[] commandObj = params.get(ApiConstants.COMMAND); - final String command = commandObj == null ? null : (String) commandObj[0]; final Object[] userObj = params.get(ApiConstants.USERNAME); String username = userObj == null ? null : (String)userObj[0]; if (LOGGER.isTraceEnabled()) { @@ -317,6 +365,19 @@ public class ApiServlet extends HttpServlet { } } + if (apiServer.isPostRequestsAndTimestampsEnforced() && !isStateChangingCommandUsingPOST(command, req.getMethod(), params)) { + String errorText = String.format("State changing command %s needs to be sent using POST request", command); + if (command.equalsIgnoreCase("updateConfiguration") && params.containsKey("name")) { + errorText = String.format("Changes for configuration %s needs to be sent using POST request", params.get("name")[0]); + } + auditTrailSb.append(" " + HttpServletResponse.SC_BAD_REQUEST + " " + errorText); + final String serializedResponse = + apiServer.getSerializedApiError(new ServerApiException(ApiErrorCode.BAD_REQUEST, errorText), params, + responseType); + HttpUtils.writeHttpResponse(resp, serializedResponse, HttpServletResponse.SC_BAD_REQUEST, responseType, ApiServer.JSONcontentType.value()); + return; + } + Long userId = null; if (!isNew) { userId = (Long)session.getAttribute("userid"); @@ -407,6 +468,15 @@ public class ApiServlet extends HttpServlet { return verify2FA; } + private boolean isStateChangingCommandUsingPOST(String command, String method, Map params) { + if (command == null || (!GET_REQUEST_COMMANDS.matcher(command.toLowerCase()).matches() && !GET_REQUEST_COMMANDS_LIST.contains(command.toLowerCase()) + && !command.equalsIgnoreCase("updateConfiguration") && !method.equals("POST"))) { + return false; + } + return !command.equalsIgnoreCase("updateConfiguration") || method.equals("POST") || (params.containsKey("name") + && params.get("name")[0].toString().equalsIgnoreCase(ApiServer.EnforcePostRequestsAndTimestamps.key())); + } + protected boolean skip2FAcheckForAPIs(String command) { boolean skip2FAcheck = false; @@ -644,4 +714,45 @@ public class ApiServlet extends HttpServlet { } return null; } + + private String getCleanParamsString(Map reqParams) { + if (MapUtils.isEmpty(reqParams)) { + return ""; + } + + StringBuilder cleanParamsString = new StringBuilder(); + for (Map.Entry reqParam : reqParams.entrySet()) { + if (org.apache.commons.lang3.StringUtils.isBlank(reqParam.getKey())) { + continue; + } + + cleanParamsString.append(reqParam.getKey()); + cleanParamsString.append("="); + + if (reqParam.getKey().toLowerCase().contains("password") + || reqParam.getKey().toLowerCase().contains("privatekey") + || reqParam.getKey().toLowerCase().contains("accesskey") + || reqParam.getKey().toLowerCase().contains("secretkey")) { + cleanParamsString.append("\n"); + continue; + } + + if (reqParam.getValue() == null || reqParam.getValue().length == 0) { + cleanParamsString.append("\n"); + continue; + } + + for (String param : reqParam.getValue()) { + if (org.apache.commons.lang3.StringUtils.isBlank(param)) { + continue; + } + String cleanParamString = StringUtils.cleanString(param.trim()); + cleanParamsString.append(cleanParamString); + cleanParamsString.append(" "); + } + cleanParamsString.append("\n"); + } + + return cleanParamsString.toString(); + } } diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index ce6f4c9cd65..091ce635596 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -318,6 +318,7 @@ import com.cloud.storage.VolumeApiServiceImpl; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.BucketDao; import com.cloud.storage.dao.DiskOfferingDao; +import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.StoragePoolTagsDao; import com.cloud.storage.dao.VMTemplateDao; @@ -648,6 +649,9 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject HostPodDao podDao; + @Inject + GuestOSDao guestOSDao; + private SearchCriteria getMinimumCpuServiceOfferingJoinSearchCriteria(int cpu) { SearchCriteria sc = _srvOfferingJoinDao.createSearchCriteria(); SearchCriteria sc1 = _srvOfferingJoinDao.createSearchCriteria(); @@ -4791,12 +4795,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } Boolean isVnf = cmd.getVnf(); + Boolean forCks = cmd.getForCks(); return searchForTemplatesInternal(id, cmd.getTemplateName(), cmd.getKeyword(), templateFilter, false, null, cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), cmd.getStoragePoolId(), cmd.getImageStoreId(), hypervisorType, showDomr, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedTmpl, cmd.getIds(), parentTemplateId, cmd.getShowUnique(), - templateType, isVnf, cmd.getArch()); + templateType, isVnf, cmd.getArch(), cmd.getOsCategoryId(), forCks); } private Pair, Integer> searchForTemplatesInternal(Long templateId, String name, String keyword, @@ -4805,7 +4810,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q boolean showDomr, boolean onlyReady, List permittedAccounts, Account caller, ListProjectResourcesCriteria listProjectResourcesCriteria, Map tags, boolean showRemovedTmpl, List ids, Long parentTemplateId, Boolean showUnique, String templateType, - Boolean isVnf, CPU.CPUArch arch) { + Boolean isVnf, CPU.CPUArch arch, Long osCategoryId, Boolean forCks) { // check if zone is configured, if not, just return empty list List hypers = null; @@ -4833,10 +4838,13 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (storagePoolId != null) { SearchBuilder storagePoolSb = templatePoolDao.createSearchBuilder(); - storagePoolSb.and("pool_id", storagePoolSb.entity().getPoolId(), SearchCriteria.Op.EQ); sb.join("storagePool", storagePoolSb, storagePoolSb.entity().getTemplateId(), sb.entity().getId(), JoinBuilder.JoinType.INNER); } + if (osCategoryId != null) { + sb.and("guestOsIdIN", sb.entity().getGuestOSId(), Op.IN); + } + SearchCriteria sc = sb.create(); if (imageStoreId != null) { @@ -4851,6 +4859,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sc.setJoinParameters("storagePool", "pool_id", storagePoolId); } + if (osCategoryId != null) { + List guestOsIds = guestOSDao.listIdsByCategoryId(osCategoryId); + if (CollectionUtils.isNotEmpty(guestOsIds)) { + sc.setParameters("guestOsIdIN", guestOsIds.toArray()); + } else { + return new Pair<>(new ArrayList<>(), 0); + } + } + // verify templateId parameter and specially handle it if (templateId != null) { template = _templateDao.findByIdIncludingRemoved(templateId); // Done for backward compatibility - Bug-5221 @@ -4991,7 +5008,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q applyPublicTemplateSharingRestrictions(sc, caller); return templateChecks(isIso, hypers, tags, name, keyword, hyperType, onlyReady, bootable, zoneId, showDomr, caller, - showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, searchFilter, sc); + showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, forCks, searchFilter, sc); } /** @@ -5045,7 +5062,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q private Pair, Integer> templateChecks(boolean isIso, List hypers, Map tags, String name, String keyword, HypervisorType hyperType, boolean onlyReady, Boolean bootable, Long zoneId, boolean showDomr, Account caller, - boolean showRemovedTmpl, Long parentTemplateId, Boolean showUnique, String templateType, Boolean isVnf, + boolean showRemovedTmpl, Long parentTemplateId, Boolean showUnique, String templateType, Boolean isVnf, Boolean forCks, Filter searchFilter, SearchCriteria sc) { if (!isIso) { // add hypervisor criteria for template case @@ -5143,6 +5160,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } } + if (forCks != null) { + sc.addAnd("forCks", SearchCriteria.Op.EQ, forCks); + } + // don't return removed template, this should not be needed since we // changed annotation for removed field in TemplateJoinVO. // sc.addAnd("removed", SearchCriteria.Op.NULL); @@ -5236,7 +5257,8 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return searchForTemplatesInternal(cmd.getId(), cmd.getIsoName(), cmd.getKeyword(), isoFilter, true, cmd.isBootable(), cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), cmd.getStoragePoolId(), cmd.getImageStoreId(), hypervisorType, true, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, - tags, showRemovedISO, null, null, cmd.getShowUnique(), null, null, cmd.getArch()); + tags, showRemovedISO, null, null, cmd.getShowUnique(), null, null, + cmd.getArch(), cmd.getOsCategoryId(), null); } @Override diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index ac24f14b781..3f983dd0cd6 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -28,54 +28,53 @@ import java.util.stream.Collectors; import javax.inject.Inject; -import com.cloud.deployasis.DeployAsIsConstants; -import com.cloud.deployasis.TemplateDeployAsIsDetailVO; -import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.VMTemplateStoragePoolVO; -import com.cloud.storage.dao.VMTemplatePoolDao; -import com.cloud.storage.VnfTemplateDetailVO; -import com.cloud.storage.VnfTemplateNicVO; -import com.cloud.storage.dao.VnfTemplateDetailsDao; -import com.cloud.storage.dao.VnfTemplateNicDao; -import com.cloud.user.dao.UserDataDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.response.VnfNicResponse; -import org.apache.cloudstack.api.response.VnfTemplateResponse; -import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; -import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; -import org.apache.cloudstack.utils.security.DigestHelper; -import org.springframework.stereotype.Component; - import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.response.ChildTemplateResponse; import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.VnfNicResponse; +import org.apache.cloudstack.api.response.VnfTemplateResponse; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateState; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.springframework.stereotype.Component; import com.cloud.api.ApiDBUtils; import com.cloud.api.ApiResponseHelper; import com.cloud.api.query.vo.ResourceTagJoinVO; import com.cloud.api.query.vo.TemplateJoinVO; +import com.cloud.deployasis.DeployAsIsConstants; +import com.cloud.deployasis.TemplateDeployAsIsDetailVO; +import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.VMTemplateStoragePoolVO; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VnfTemplateDetailVO; +import com.cloud.storage.VnfTemplateNicVO; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.storage.dao.VMTemplateDetailsDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VnfTemplateDetailsDao; +import com.cloud.storage.dao.VnfTemplateNicDao; import com.cloud.template.VirtualMachineTemplate; import com.cloud.user.Account; import com.cloud.user.AccountService; +import com.cloud.user.dao.UserDataDao; import com.cloud.utils.Pair; import com.cloud.utils.db.Filter; import com.cloud.utils.db.SearchBuilder; @@ -262,6 +261,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation weightBasedParametersForValidation = new HashSet<>(); private Set overprovisioningFactorsForValidation = new HashSet<>(); - public static final ConfigKey SystemVMUseLocalStorage = new ConfigKey(Boolean.class, "system.vm.use.local.storage", "Advanced", "false", + public static final ConfigKey SystemVMUseLocalStorage = new ConfigKey<>(Boolean.class, "system.vm.use.local.storage", "Advanced", "false", "Indicates whether to use local storage pools or shared storage pools for system VMs.", false, ConfigKey.Scope.Zone, null); - public final static ConfigKey BYTES_MAX_READ_LENGTH= new ConfigKey(Long.class, "vm.disk.bytes.maximum.read.length", "Advanced", "0", + public final static ConfigKey BYTES_MAX_READ_LENGTH= new ConfigKey<>(Long.class, "vm.disk.bytes.maximum.read.length", "Advanced", "0", "Maximum Bytes read burst duration (seconds). If '0' (zero) then does not check for maximum burst length.", true, ConfigKey.Scope.Global, null); - public final static ConfigKey BYTES_MAX_WRITE_LENGTH = new ConfigKey(Long.class, "vm.disk.bytes.maximum.write.length", "Advanced", "0", + public final static ConfigKey BYTES_MAX_WRITE_LENGTH = new ConfigKey<>(Long.class, "vm.disk.bytes.maximum.write.length", "Advanced", "0", "Maximum Bytes write burst duration (seconds). If '0' (zero) then does not check for maximum burst length.", true, ConfigKey.Scope.Global, null); - public final static ConfigKey IOPS_MAX_READ_LENGTH = new ConfigKey(Long.class, "vm.disk.iops.maximum.read.length", "Advanced", "0", + public final static ConfigKey IOPS_MAX_READ_LENGTH = new ConfigKey<>(Long.class, "vm.disk.iops.maximum.read.length", "Advanced", "0", "Maximum IOPS read burst duration (seconds). If '0' (zero) then does not check for maximum burst length.", true, ConfigKey.Scope.Global, null); - public final static ConfigKey IOPS_MAX_WRITE_LENGTH = new ConfigKey(Long.class, "vm.disk.iops.maximum.write.length", "Advanced", "0", + public final static ConfigKey IOPS_MAX_WRITE_LENGTH = new ConfigKey<>(Long.class, "vm.disk.iops.maximum.write.length", "Advanced", "0", "Maximum IOPS write burst duration (seconds). If '0' (zero) then does not check for maximum burst length.", true, ConfigKey.Scope.Global, null); - public static final ConfigKey ADD_HOST_ON_SERVICE_RESTART_KVM = new ConfigKey(Boolean.class, "add.host.on.service.restart.kvm", "Advanced", "true", + public static final ConfigKey ADD_HOST_ON_SERVICE_RESTART_KVM = new ConfigKey<>(Boolean.class, "add.host.on.service.restart.kvm", "Advanced", "true", "Indicates whether the host will be added back to cloudstack after restarting agent service on host. If false it won't be added back even after service restart", true, ConfigKey.Scope.Global, null); - public static final ConfigKey SET_HOST_DOWN_TO_MAINTENANCE = new ConfigKey(Boolean.class, "set.host.down.to.maintenance", "Advanced", "false", - "Indicates whether the host in down state can be put into maintenance state so thats its not enabled after it comes back.", - true, ConfigKey.Scope.Zone, null); - public static final ConfigKey ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN = new ConfigKey(Boolean.class, "enable.account.settings.for.domain", "Advanced", "false", + public static final ConfigKey SET_HOST_DOWN_TO_MAINTENANCE = new ConfigKey<>(Boolean.class, "set.host.down.to.maintenance", "Advanced", "false", + "Indicates whether the host in down state can be put into maintenance state so thats its not enabled after it comes back.", + true, ConfigKey.Scope.Zone, null); + public static final ConfigKey ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN = new ConfigKey<>(Boolean.class, "enable.account.settings.for.domain", "Advanced", "false", "Indicates whether to add account settings for domain. If true, account settings will be added to domain settings, all accounts in the domain will inherit the domain setting if account setting is not set.", true, ConfigKey.Scope.Global, null); - public static final ConfigKey ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN = new ConfigKey(Boolean.class, "enable.domain.settings.for.child.domain", "Advanced", "false", + public static final ConfigKey ENABLE_DOMAIN_SETTINGS_FOR_CHILD_DOMAIN = new ConfigKey<>(Boolean.class, "enable.domain.settings.for.child.domain", "Advanced", "false", "Indicates whether the settings of parent domain should be applied for child domain. If true, the child domain will get value from parent domain if its not configured in child domain else global value is taken.", true, ConfigKey.Scope.Global, null); - public static ConfigKey VM_SERVICE_OFFERING_MAX_CPU_CORES = new ConfigKey("Advanced", Integer.class, "vm.serviceoffering.cpu.cores.max", "0", "Maximum CPU cores " - + "for vm service offering. If 0 - no limitation", true); + public static ConfigKey VM_SERVICE_OFFERING_MAX_CPU_CORES = new ConfigKey<>("Advanced", Integer.class, "vm.serviceoffering.cpu.cores.max", "0", "Maximum CPU cores " + + "for vm service offering. If 0 - no limitation", true); - public static ConfigKey VM_SERVICE_OFFERING_MAX_RAM_SIZE = new ConfigKey("Advanced", Integer.class, "vm.serviceoffering.ram.size.max", "0", "Maximum RAM size in " - + "MB for vm service offering. If 0 - no limitation", true); + public static ConfigKey VM_SERVICE_OFFERING_MAX_RAM_SIZE = new ConfigKey<>("Advanced", Integer.class, "vm.serviceoffering.ram.size.max", "0", "Maximum RAM size in " + + "MB for vm service offering. If 0 - no limitation", true); - public static final ConfigKey MIGRATE_VM_ACROSS_CLUSTERS = new ConfigKey(Boolean.class, "migrate.vm.across.clusters", "Advanced", "false", - "Indicates whether the VM can be migrated to different cluster if no host is found in same cluster",true, ConfigKey.Scope.Zone, null); + public static final ConfigKey MIGRATE_VM_ACROSS_CLUSTERS = new ConfigKey<>(Boolean.class, "migrate.vm.across.clusters", "Advanced", "false", + "Indicates whether the VM can be migrated to different cluster if no host is found in same cluster", true, ConfigKey.Scope.Zone, null); public static final ConfigKey ALLOW_DOMAIN_ADMINS_TO_CREATE_TAGGED_OFFERINGS = new ConfigKey<>(Boolean.class, "allow.domain.admins.to.create.tagged.offerings", "Advanced", "false", "Allow domain admins to create offerings with tags.", true, ConfigKey.Scope.Account, null); @@ -539,7 +526,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati private static final String BYTES_WRITE_RATE = "Bytes Write"; private static final String DefaultForSystemVmsForPodIpRange = "0"; - private static final String DefaultVlanForPodIpRange = Vlan.UNTAGGED.toString(); + private static final String DefaultVlanForPodIpRange = Vlan.UNTAGGED; private static final Set VPC_ONLY_PROVIDERS = Sets.newHashSet(Provider.VPCVirtualRouter, Provider.JuniperContrailVpcRouter, Provider.InternalLbVm); @@ -630,21 +617,30 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati private void initMessageBusListener() { messageBus.subscribe(EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, new MessageSubscriber() { @Override - public void onPublishMessage(String serderAddress, String subject, Object args) { - String globalSettingUpdated = (String) args; - if (StringUtils.isEmpty(globalSettingUpdated)) { + public void onPublishMessage(String senderAddress, String subject, Object args) { + Ternary settingUpdated = (Ternary) args; + String settingNameUpdated = settingUpdated.first(); + if (StringUtils.isEmpty(settingNameUpdated)) { return; } - if (globalSettingUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) || - globalSettingUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) { - _indirectAgentLB.propagateMSListToAgents(); - } else if (globalSettingUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString()) - || globalSettingUpdated.equals(Config.MigrateWait.toString())) { - Map params = new HashMap(); + if (settingNameUpdated.equals(ApiServiceConfiguration.ManagementServerAddresses.key()) || + settingNameUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBAlgorithm.key())) { + _indirectAgentLB.propagateMSListToAgents(false); + } else if (settingNameUpdated.equals(Config.RouterAggregationCommandEachTimeout.toString()) + || settingNameUpdated.equals(Config.MigrateWait.toString())) { + Map params = new HashMap<>(); params.put(Config.RouterAggregationCommandEachTimeout.toString(), _configDao.getValue(Config.RouterAggregationCommandEachTimeout.toString())); params.put(Config.MigrateWait.toString(), _configDao.getValue(Config.MigrateWait.toString())); _agentManager.propagateChangeToAgents(params); - } else if (VMLeaseManager.InstanceLeaseEnabled.key().equals(globalSettingUpdated)) { + } else if (settingNameUpdated.equals(IndirectAgentLBServiceImpl.IndirectAgentLBCheckInterval.key())) { + ConfigKey.Scope scope = settingUpdated.second(); + if (scope == ConfigKey.Scope.Global) { + _indirectAgentLB.propagateMSListToAgents(false); + } else if (scope == ConfigKey.Scope.Cluster) { + Long clusterId = settingUpdated.third(); + _indirectAgentLB.propagateMSListToAgentsInCluster(clusterId); + } + } else if (VMLeaseManager.InstanceLeaseEnabled.key().equals(settingNameUpdated)) { vmLeaseManager.onLeaseFeatureToggle(); } } @@ -677,6 +673,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } + protected void validateConflictingConfigValue(final String configName, final String value) { + if (configName.equals("cloud.kubernetes.etcd.node.start.port")) { + if (value.equals(CLUSTER_NODES_DEFAULT_START_SSH_PORT)) { + String errorMessage = "This range is reserved for Kubernetes cluster nodes." + + "Please choose a value in a higher range would does not conflict with a kubernetes cluster deployed"; + logger.error(errorMessage); + throw new InvalidParameterValueException(errorMessage); + } + } + } + @Override public boolean start() { @@ -691,14 +698,14 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (mgtCidr == null || mgtCidr.trim().isEmpty()) { final String[] localCidrs = NetUtils.getLocalCidrs(); if (localCidrs != null && localCidrs.length > 0) { - logger.warn("Management network CIDR is not configured originally. Set it default to " + localCidrs[0]); + logger.warn("Management network CIDR is not configured originally. Set it default to {}", localCidrs[0]); - _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_MANAGEMENT_NODE, 0, new Long(0), "Management network CIDR is not configured originally. Set it default to " + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_MANAGEMENT_NODE, 0, 0L, "Management network CIDR is not configured originally. Set it default to " + localCidrs[0], ""); _configDao.update(Config.ManagementNetwork.key(), Config.ManagementNetwork.getCategory(), localCidrs[0]); } else { logger.warn("Management network CIDR is not properly configured and we are not able to find a default setting"); - _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_MANAGEMENT_NODE, 0, new Long(0), + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_MANAGEMENT_NODE, 0, 0L, "Management network CIDR is not properly configured and we are not able to find a default setting", ""); } } @@ -706,14 +713,13 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return true; } - @Override - public boolean stop() { - return true; - } - @Override @DB public String updateConfiguration(final long userId, final String name, final String category, String value, ConfigKey.Scope scope, final Long resourceId) { + if (Boolean.class == getConfigurationTypeWrapperClass(name)) { + value = value.toLowerCase(); + } + final String validationMsg = validateConfigurationValue(name, value, scope); if (validationMsg != null) { logger.error("Invalid value [{}] for configuration [{}] due to [{}].", value, name, validationMsg); @@ -833,6 +839,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati CallContext.current().setEventDetails(String.format(" Name: %s, New Value: %s, Scope: %s", name, value, scope.name())); _configDepot.invalidateConfigCache(name, scope, resourceId); + messageBus.publish(_name, EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, PublishScope.GLOBAL, new Ternary<>(name, scope, resourceId)); return valueEncrypted ? DBEncryptionUtil.decrypt(value) : value; } @@ -842,12 +849,12 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati String previousValue = _configDao.getValue(name); if (!_configDao.update(name, category, value)) { - logger.error("Failed to update configuration option, name: " + name + ", value:" + value); + logger.error("Failed to update configuration option, name: {}, value: {}", name, value); throw new CloudRuntimeException("Failed to update configuration value. Please contact Cloud Support."); } _configDepot.invalidateConfigCache(name, ConfigKey.Scope.Global, null); - PreparedStatement pstmt = null; + PreparedStatement pstmt; if (Config.XenServerGuestNetwork.key().equalsIgnoreCase(name)) { final String sql = "update host_details set value=? where name=?"; try { @@ -911,11 +918,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati try { // Change for templates pstmt = txn.prepareAutoCloseStatement(sqlTemplate); - pstmt.setDate(1, new Date(-1l));// Set the time before the epoch time. + pstmt.setDate(1, new Date(-1L));// Set the time before the epoch time. pstmt.executeUpdate(); // Change for volumes pstmt = txn.prepareAutoCloseStatement(sqlVolume); - pstmt.setDate(1, new Date(-1l));// Set the time before the epoch time. + pstmt.setDate(1, new Date(-1L));// Set the time before the epoch time. pstmt.executeUpdate(); // Cleanup the download urls _storageManager.cleanupDownloadUrls(); @@ -927,7 +934,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } txn.commit(); - messageBus.publish(_name, EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, PublishScope.GLOBAL, name); + messageBus.publish(_name, EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, PublishScope.GLOBAL, new Ternary<>(name, ConfigKey.Scope.Global, resourceId)); return _configDao.getValue(name); } @@ -943,8 +950,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati String hypervisors = _configDao.getValue(hypervisorListConfigName); if (Arrays.asList(hypervisors.split(",")).contains(previousValue)) { hypervisors = hypervisors.replace(previousValue, newValue); - logger.info(String.format("Updating the hypervisor list configuration '%s' " + - "to match the new custom hypervisor display name", hypervisorListConfigName)); + logger.info("Updating the hypervisor list configuration '{}}' to match the new custom hypervisor display name", + hypervisorListConfigName); _configDao.update(hypervisorListConfigName, hypervisors); } } @@ -952,7 +959,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati @Override @ActionEvent(eventType = EventTypes.EVENT_CONFIGURATION_VALUE_EDIT, eventDescription = "updating configuration") public Configuration updateConfiguration(final UpdateCfgCmd cmd) throws InvalidParameterValueException { - final Long userId = CallContext.current().getCallingUserId(); + final long userId = CallContext.current().getCallingUserId(); final String name = cmd.getCfgName(); String value = cmd.getValue(); final Long zoneId = cmd.getZoneId(); @@ -981,7 +988,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // FIX ME - All configuration parameters are not moved from config.java to configKey if (config == null) { if (_configDepot.get(name) == null) { - logger.warn("Probably the component manager where configuration variable " + name + " is defined needs to implement Configurable interface"); + logger.warn("Probably the component manager where configuration variable {} is defined needs to implement Configurable interface", name); throw new InvalidParameterValueException("Config parameter with name " + name + " doesn't exist"); } category = _configDepot.get(name).category(); @@ -989,8 +996,11 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati category = config.getCategory(); } + validateIpAddressRelatedConfigValues(name, value); + validateConflictingConfigValue(name, value); + if (CATEGORY_SYSTEM.equals(category) && !_accountMgr.isRootAdmin(caller.getId())) { - logger.warn("Only Root Admin is allowed to edit the configuration " + name); + logger.warn("Only Root Admin is allowed to edit the configuration {}", name); throw new CloudRuntimeException("Only Root Admin is allowed to edit this configuration."); } @@ -1103,7 +1113,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (config == null) { configKey = _configDepot.get(name); if (configKey == null) { - logger.warn("Probably the component manager where configuration variable " + name + " is defined needs to implement Configurable interface"); + logger.warn("Probably the component manager where configuration variable {} is defined needs to implement Configurable interface", name); throw new InvalidParameterValueException("Config parameter with name " + name + " doesn't exist"); } defaultValue = configKey.defaultValue(); @@ -1225,7 +1235,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati default: if (!_configDao.update(name, category, defaultValue)) { - logger.error("Failed to reset configuration option, name: " + name + ", defaultValue:" + defaultValue); + logger.error("Failed to reset configuration option, name: {}, defaultValue: {}", name, defaultValue); throw new CloudRuntimeException("Failed to reset configuration value. Please contact Cloud Support."); } optionalValue = Optional.ofNullable(configKey != null ? configKey.value() : _configDao.findByName(name).getValue()); @@ -1239,7 +1249,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati _configDepot.invalidateConfigCache(name, scope, id); CallContext.current().setEventDetails(" Name: " + name + " New Value: " + (name.toLowerCase().contains("password") ? "*****" : defaultValue == null ? "" : defaultValue)); - return new Pair(_configDao.findByName(name), newValue); + return new Pair<>(_configDao.findByName(name), newValue); } private String getConfigurationValueInScope(ConfigurationVO config, String name, ConfigKey.Scope scope, Long id) { @@ -1264,7 +1274,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati protected String validateConfigurationValue(String name, String value, ConfigKey.Scope scope) { final ConfigurationVO cfg = _configDao.findByName(name); if (cfg == null) { - logger.error("Missing configuration variable " + name + " in configuration table"); + logger.error("Missing configuration variable {} in configuration table", name); return "Invalid configuration variable."; } validateConfigurationAllowedOnlyForDefaultAdmin(name, value); @@ -1274,22 +1284,13 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (!configScope.contains(scope) && !(ENABLE_ACCOUNT_SETTINGS_FOR_DOMAIN.value() && configScope.contains(ConfigKey.Scope.Account) && ConfigKey.Scope.Domain.equals(scope))) { - logger.error("Invalid scope id provided for the parameter " + name); + logger.error("Invalid scope id provided for the parameter {}", name); return "Invalid scope id provided for the parameter " + name; } } - Class type; - Config configuration = Config.getConfig(name); - if (configuration == null) { - logger.warn("Did not find configuration " + name + " in Config.java. Perhaps moved to ConfigDepot"); - ConfigKey configKey = _configDepot.get(name); - if(configKey == null) { - logger.warn("Did not find configuration " + name + " in ConfigDepot too."); - return null; - } - type = configKey.type(); - } else { - type = configuration.getType(); + Class type = getConfigurationTypeWrapperClass(name); + if (type == null) { + return null; } validateSpecificConfigurationValues(name, value, type); @@ -1299,7 +1300,28 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return String.format("Value [%s] is not a valid [%s].", value, type); } - return validateValueRange(name, value, type, configuration); + return validateValueRange(name, value, type, Config.getConfig(name)); + } + + /** + * Returns the configuration type's wrapper class. + * @param name name of the configuration. + * @return if the configuration exists, returns its type's wrapper class; if not, returns null. + */ + protected Class getConfigurationTypeWrapperClass(String name) { + Config configuration = Config.getConfig(name); + if (configuration != null) { + return configuration.getType(); + } + + logger.warn("Did not find configuration [{}] in Config.java. Perhaps moved to ConfigDepot.", name); + ConfigKey configKey = _configDepot.get(name); + if (configKey == null) { + logger.warn("Did not find configuration [{}] in ConfigDepot too.", name); + return null; + } + + return configKey.type(); } protected void validateConfigurationAllowedOnlyForDefaultAdmin(String configName, String value) { @@ -1336,7 +1358,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati *
    *
  • String: any value, including null;
    • *
  • Character: any value, including null;
    • - *
  • Boolean: strings that equal "true" or "false" (case-sensitive);
    • + *
  • Boolean: strings that equal "true" or "false" (case-insensitive);
    • *
  • Integer, Short, Long: strings that contain a valid int/short/long;
    • *
  • Float, Double: strings that contain a valid float/double, except infinity.
    • *
@@ -1558,7 +1580,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final int max = Integer.parseInt(options[1]); final int val = Integer.parseInt(value); if (val < min || val > max) { - logger.error(String.format("Invalid value for configuration [%s]. Please enter a value in the range [%s].", name, range)); + logger.error("Invalid value for configuration [{}]. Please enter a value in the range [{}].", name, range); return String.format("The provided value is not valid for this configuration. Please enter an integer in the range: [%s]", range); } return null; @@ -1568,7 +1590,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati * Checks if the value for the configuration is valid for any of the ranges selected. */ protected String validateIfStringValueIsInRange(String name, String value, String... range) { - List message = new ArrayList(); + List message = new ArrayList<>(); String errMessage = ""; for (String rangeOption : range) { switch (rangeOption) { @@ -1606,9 +1628,9 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (NetUtils.isSiteLocalAddress(value)) { return null; } - logger.error(String.format("Value [%s] is not a valid private IP range for configuration [%s].", value, name)); + logger.error("Value [{}] is not a valid private IP range for configuration [{}].", value, name); } catch (final NullPointerException e) { - logger.error(String.format("Error while parsing IP address for [%s].", name)); + logger.error("Error while parsing IP address for [{}].", name); } return "a valid site local IP address"; } @@ -1664,7 +1686,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return null; } } - logger.error(String.format("Invalid value for configuration [%s].", name)); + logger.error("Invalid value for configuration [{}].", name); return String.format("a valid value for this configuration (Options are: [%s])", rangeOption); } @@ -1880,7 +1902,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } String vlanNumberFromUri = getVlanNumberFromUri(vlan); - final Integer vlanId = vlanNumberFromUri.equals(Vlan.UNTAGGED.toString()) ? null : Integer.parseInt(vlanNumberFromUri); + final Integer vlanId = vlanNumberFromUri.equals(Vlan.UNTAGGED) ? null : Integer.parseInt(vlanNumberFromUri); final HostPodVO pod = _podDao.findById(podId); @@ -1992,7 +2014,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } }); } catch (final Exception e) { - logger.error("Unable to create Pod IP range due to " + e.getMessage(), e); + logger.error("Unable to create Pod IP range due to {}", e.getMessage(), e); throw new CloudRuntimeException("Failed to create Pod IP range. Please contact Cloud Support."); } @@ -2310,7 +2332,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } DataCenterGuestIpv6Prefix dataCenterGuestIpv6Prefix = null; try { - dataCenterGuestIpv6Prefix = Transaction.execute(new TransactionCallback() { + dataCenterGuestIpv6Prefix = Transaction.execute(new TransactionCallback<>() { @Override public DataCenterGuestIpv6Prefix doInTransaction(TransactionStatus status) { DataCenterGuestIpv6PrefixVO dataCenterGuestIpv6PrefixVO = new DataCenterGuestIpv6PrefixVO(zoneId, prefix); @@ -2319,7 +2341,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } }); } catch (final Exception e) { - logger.error(String.format("Unable to add IPv6 prefix for zone: %s due to %s", zone, e.getMessage()), e); + logger.error("Unable to add IPv6 prefix for zone: {} due to {}", zone, e.getMessage(), e); throw new CloudRuntimeException(String.format("Unable to add IPv6 prefix for zone ID: %s. Please contact Cloud Support.", zone)); } return dataCenterGuestIpv6Prefix; @@ -2476,7 +2498,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati messageBus.publish(_name, MESSAGE_DELETE_POD_IP_RANGE_EVENT, PublishScope.LOCAL, pod); messageBus.publish(_name, MESSAGE_CREATE_POD_IP_RANGE_EVENT, PublishScope.LOCAL, pod); } catch (final Exception e) { - logger.error("Unable to edit pod due to " + e.getMessage(), e); + logger.error("Unable to edit pod due to {}", e.getMessage(), e); throw new CloudRuntimeException("Failed to edit pod. Please contact Cloud Support."); } @@ -2771,7 +2793,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati checkIfZoneIsDeletable(zoneId); - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public Boolean doInTransaction(final TransactionStatus status) { // delete vlans for this zone @@ -2835,7 +2857,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final String networkDomain = cmd.getDomain(); final Boolean localStorageEnabled = cmd.getLocalStorageEnabled(); - final Map newDetails = new HashMap(); + final Map newDetails = new HashMap<>(); if (detailsMap != null) { final Collection zoneDetailsCollection = detailsMap.values(); final Iterator iter = zoneDetailsCollection.iterator(); @@ -2950,7 +2972,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(final TransactionStatus status) { - final Map updatedDetails = new HashMap(); + final Map updatedDetails = new HashMap<>(); _zoneDao.loadDetails(zone); if (zone.getDetails() != null) { updatedDetails.putAll(zone.getDetails()); @@ -3069,7 +3091,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati zoneFinal.setStorageAccessGroups(String.join(",", storageAccessGroups)); } - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public DataCenterVO doInTransaction(final TransactionStatus status) { final DataCenterVO zone = _zoneDao.persist(zoneFinal); @@ -3457,7 +3479,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati limitResourceUse, volatileVm, displayText, isSystem, vmType, hostTag, deploymentPlanner, dynamicScalingEnabled, isCustomized); - List detailsVOList = new ArrayList(); + List detailsVOList = new ArrayList<>(); if (details != null) { // To have correct input, either both gpu card name and VGPU type should be passed or nothing should be passed. // Use XOR condition to verify that. @@ -4163,28 +4185,28 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati * */ protected void validateMaximumIopsAndBytesLength(final Long iopsReadRateMaxLength, final Long iopsWriteRateMaxLength, Long bytesReadRateMaxLength, Long bytesWriteRateMaxLength) { - if (IOPS_MAX_READ_LENGTH.value() != null && IOPS_MAX_READ_LENGTH.value() != 0l) { + if (IOPS_MAX_READ_LENGTH.value() != null && IOPS_MAX_READ_LENGTH.value() != 0L) { if (iopsReadRateMaxLength != null && iopsReadRateMaxLength > IOPS_MAX_READ_LENGTH.value()) { throw new InvalidParameterValueException(String.format("IOPS read max length (%d seconds) cannot be greater than vm.disk.iops.maximum.read.length (%d seconds)", iopsReadRateMaxLength, IOPS_MAX_READ_LENGTH.value())); } } - if (IOPS_MAX_WRITE_LENGTH.value() != null && IOPS_MAX_WRITE_LENGTH.value() != 0l) { + if (IOPS_MAX_WRITE_LENGTH.value() != null && IOPS_MAX_WRITE_LENGTH.value() != 0L) { if (iopsWriteRateMaxLength != null && iopsWriteRateMaxLength > IOPS_MAX_WRITE_LENGTH.value()) { throw new InvalidParameterValueException(String.format("IOPS write max length (%d seconds) cannot be greater than vm.disk.iops.maximum.write.length (%d seconds)", iopsWriteRateMaxLength, IOPS_MAX_WRITE_LENGTH.value())); } } - if (BYTES_MAX_READ_LENGTH.value() != null && BYTES_MAX_READ_LENGTH.value() != 0l) { + if (BYTES_MAX_READ_LENGTH.value() != null && BYTES_MAX_READ_LENGTH.value() != 0L) { if (bytesReadRateMaxLength != null && bytesReadRateMaxLength > BYTES_MAX_READ_LENGTH.value()) { throw new InvalidParameterValueException(String.format("Bytes read max length (%d seconds) cannot be greater than vm.disk.bytes.maximum.read.length (%d seconds)", bytesReadRateMaxLength, BYTES_MAX_READ_LENGTH.value())); } } - if (BYTES_MAX_WRITE_LENGTH.value() != null && BYTES_MAX_WRITE_LENGTH.value() != 0l) { + if (BYTES_MAX_WRITE_LENGTH.value() != null && BYTES_MAX_WRITE_LENGTH.value() != 0L) { if (bytesWriteRateMaxLength != null && bytesWriteRateMaxLength > BYTES_MAX_WRITE_LENGTH.value()) { throw new InvalidParameterValueException(String.format("Bytes write max length (%d seconds) cannot be greater than vm.disk.bytes.maximum.write.length (%d seconds)", bytesWriteRateMaxLength, BYTES_MAX_WRITE_LENGTH.value())); @@ -4932,7 +4954,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati commitVlanLock.lock(5); logger.debug("Acquiring lock for committing vlan"); try { - Vlan vlan = Transaction.execute(new TransactionCallback() { + Vlan vlan = Transaction.execute(new TransactionCallback<>() { @Override public Vlan doInTransaction(final TransactionStatus status) { String newVlanNetmask = newVlanNetmaskFinal; @@ -4941,10 +4963,10 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if ((sameSubnet == null || !sameSubnet.first()) && network.getTrafficType() == TrafficType.Guest && network.getGuestType() == GuestType.Shared && _vlanDao.listVlansByNetworkId(networkId) != null) { final Map dhcpCapabilities = _networkSvc.getNetworkOfferingServiceCapabilities(_networkOfferingDao.findById(network.getNetworkOfferingId()), - Service.Dhcp); + Service.Dhcp); final String supportsMultipleSubnets = dhcpCapabilities.get(Capability.DhcpAccrossMultipleSubnets); if (supportsMultipleSubnets == null || !Boolean.valueOf(supportsMultipleSubnets)) { - throw new InvalidParameterValueException("The dhcp service provider for this network does not support dhcp across multiple subnets"); + throw new InvalidParameterValueException("The dhcp service provider for this network does not support dhcp across multiple subnets"); } logger.info("adding a new subnet to the network {}", network); } else if (sameSubnet != null) { @@ -5023,16 +5045,16 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati Pair vlanDetails = null; if (sameSubnet) { - vlanDetails = new Pair(vlanGateway, vlanNetmask); + vlanDetails = new Pair<>(vlanGateway, vlanNetmask); } else { - vlanDetails = new Pair(newVlanGateway, newVlanNetmask); + vlanDetails = new Pair<>(newVlanGateway, newVlanNetmask); } // check if the gatewayip is the part of the ip range being added. if (ipv4 && NetUtils.ipRangesOverlap(startIP, endIP, vlanDetails.first(), vlanDetails.first())) { throw new InvalidParameterValueException("The gateway ip should not be the part of the ip range being added."); } - return new Pair>(sameSubnet, vlanDetails); + return new Pair<>(sameSubnet, vlanDetails); } public boolean hasSameSubnet(boolean ipv4, String vlanGateway, String vlanNetmask, String newVlanGateway, String newVlanNetmask, String newStartIp, String newEndIp, @@ -5363,7 +5385,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati private VlanVO commitVlanAndIpRange(final long zoneId, final long networkId, final long physicalNetworkId, final Long podId, final String startIP, final String endIP, final String vlanGateway, final String vlanNetmask, final String vlanId, final Domain domain, final Account vlanOwner, final String vlanIp6Gateway, final String vlanIp6Cidr, final boolean ipv4, final DataCenterVO zone, final VlanType vlanType, final String ipv6Range, final String ipRange, final boolean forSystemVms, final boolean forNsx) { - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public VlanVO doInTransaction(final TransactionStatus status) { VlanVO vlan = new VlanVO(vlanType, vlanId, vlanGateway, vlanNetmask, zone.getId(), ipRange, networkId, physicalNetworkId, vlanIp6Gateway, vlanIp6Cidr, ipv6Range); @@ -5598,7 +5620,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final String gateway, final String netmask, final boolean ipv4, final Boolean isRangeForSystemVM, final Boolean forSystemvms) { - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public VlanVO doInTransaction(final TransactionStatus status) { VlanVO vlanRange = _vlanDao.findById(id); @@ -5950,7 +5972,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final long allocIpCount = _publicIpAddressDao.countIPs(vlan.getDataCenterId(), vlanDbId, true); final List ips = _publicIpAddressDao.listByVlanId(vlanDbId); boolean success = true; - final List ipsInUse = new ArrayList(); + final List ipsInUse = new ArrayList<>(); if (allocIpCount > 0) { try { vlan = _vlanDao.acquireInLockTable(vlanDbId, 30); @@ -6007,7 +6029,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final long startIPLong = NetUtils.ip2Long(startIP); final long endIPLong = NetUtils.ip2Long(endIP); - final List problemIps = Transaction.execute(new TransactionCallback>() { + final List problemIps = Transaction.execute(new TransactionCallback<>() { @Override public List doInTransaction(final TransactionStatus status) { final IPRangeConfig config = new IPRangeConfig(); @@ -6036,7 +6058,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati currentStartIPLong++; } - final List problemIps = Transaction.execute(new TransactionCallback>() { + final List problemIps = Transaction.execute(new TransactionCallback<>() { @Override public List doInTransaction(final TransactionStatus status) { @@ -6156,7 +6178,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati skipPod = podIdToBeSkipped; } final HashMap> currentPodCidrSubnets = _podDao.getCurrentPodCidrSubnets(dcId, skipPod); - final List newCidrPair = new ArrayList(); + final List newCidrPair = new ArrayList<>(); newCidrPair.add(0, getCidrAddress(cidr)); newCidrPair.add(1, (long)getCidrSize(cidr)); currentPodCidrSubnets.put(new Long(-1), newCidrPair); @@ -6468,8 +6490,8 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati NetworkOffering.RoutingMode routingMode = verifyRoutingMode(routingModeString); // configure service provider map - final Map> serviceProviderMap = new HashMap>(); - final Set defaultProviders = new HashSet(); + final Map> serviceProviderMap = new HashMap<>(); + final Set defaultProviders = new HashSet<>(); // populate the services first for (final String serviceName : cmd.getSupportedServices()) { @@ -6492,7 +6514,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (guestType != GuestType.Shared) { throw new InvalidParameterValueException("Security group service is supported for network offerings with guest ip type " + GuestType.Shared); } - final Set sgProviders = new HashSet(); + final Set sgProviders = new HashSet<>(); sgProviders.add(Provider.SecurityGroupProvider); serviceProviderMap.put(Network.Service.SecurityGroup, sgProviders); continue; @@ -6508,7 +6530,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } // populate providers - final Map> providerCombinationToVerify = new HashMap>(); + final Map> providerCombinationToVerify = new HashMap<>(); final Map> svcPrv = cmd.getServiceProviders(); Provider firewallProvider = null; Provider dhcpProvider = null; @@ -6517,7 +6539,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati for (final String serviceStr : svcPrv.keySet()) { final Network.Service service = Network.Service.getService(serviceStr); if (serviceProviderMap.containsKey(service)) { - final Set providers = new HashSet(); + final Set providers = new HashSet<>(); // Allow to specify more than 1 provider per service only if // the service is LB if (!serviceStr.equalsIgnoreCase(Service.Lb.getName()) && svcPrv.get(serviceStr) != null && svcPrv.get(serviceStr).size() > 1) { @@ -6562,7 +6584,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati Set serviceSet = null; if (providerCombinationToVerify.get(provider) == null) { - serviceSet = new HashSet(); + serviceSet = new HashSet<>(); } else { serviceSet = providerCombinationToVerify.get(provider); } @@ -6630,7 +6652,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } validateConnectivityServiceCapablities(guestType, serviceProviderMap.get(Service.Connectivity), connectivityServiceCapabilityMap); - final Map> serviceCapabilityMap = new HashMap>(); + final Map> serviceCapabilityMap = new HashMap<>(); serviceCapabilityMap.put(Service.Lb, lbServiceCapabilityMap); serviceCapabilityMap.put(Service.SourceNat, sourceNatServiceCapabilityMap); serviceCapabilityMap.put(Service.StaticNat, staticNatServiceCapabilityMap); @@ -6645,7 +6667,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // combination if (firewallProvider != null) { logger.debug("Adding Firewall service with provider " + firewallProvider.getName()); - final Set firewallProviderSet = new HashSet(); + final Set firewallProviderSet = new HashSet<>(); firewallProviderSet.add(firewallProvider); serviceProviderMap.put(Service.Firewall, firewallProviderSet); if (!(firewallProvider.getName().equals(Provider.JuniperSRX.getName()) || firewallProvider.getName().equals(Provider.PaloAlto.getName()) || firewallProvider.getName() @@ -6655,7 +6677,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } - final Map details = new HashMap(); + final Map details = new HashMap<>(); if (detailsStr != null) { for (final String detailStr : detailsStr.keySet()) { NetworkOffering.Detail offDetail = null; @@ -7119,7 +7141,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } - return Transaction.execute(new TransactionCallback() { + return Transaction.execute(new TransactionCallback<>() { @Override public NetworkOfferingVO doInTransaction(final TransactionStatus status) { NetworkOfferingVO offering = offeringFinal; @@ -7145,7 +7167,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } if (vpcOff && !forNsx) { - final List supportedSvcs = new ArrayList(); + final List supportedSvcs = new ArrayList<>(); supportedSvcs.addAll(serviceProviderMap.keySet()); _vpcMgr.validateNtwkOffForVpc(offering, supportedSvcs); } @@ -7229,7 +7251,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } // 2) validate if the provider supports the scheme - final Set lbProviders = new HashSet(); + final Set lbProviders = new HashSet<>(); lbProviders.add(lbProvider); if (detail == NetworkOffering.Detail.InternalLbProvider) { _networkModel.checkCapabilityForProvider(lbProviders, Service.Lb, Capability.LbSchemes, Scheme.Internal.toString()); @@ -7335,7 +7357,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (zone.getNetworkType() == NetworkType.Basic) { // return empty list as we don't allow to create networks in // basic zone, and shouldn't display networkOfferings - return new Pair, Integer>(new ArrayList(), 0); + return new Pair<>(new ArrayList<>(), 0); } } @@ -7368,7 +7390,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati if (!offeringIds.isEmpty()) { sc.addAnd("id", SearchCriteria.Op.IN, offeringIds.toArray()); } else { - return new Pair, Integer>(new ArrayList(), 0); + return new Pair<>(new ArrayList<>(), 0); } } @@ -7420,7 +7442,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati } } final Boolean sourceNatSupported = cmd.getSourceNatSupported(); - final List pNtwkTags = new ArrayList(); + final List pNtwkTags = new ArrayList<>(); boolean checkForTags = false; boolean allowNullTag = false; if (zone != null) { @@ -7453,7 +7475,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati for (final NetworkOfferingJoinVO offering : offerings) { boolean addOffering = true; - List checkForProviders = new ArrayList(); + List checkForProviders = new ArrayList<>(); if (checkForTags && !checkNetworkOfferingTags(pNtwkTags, allowNullTag, offering.getTags())) { continue; @@ -7492,17 +7514,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati // Now apply pagination final List wPagination = com.cloud.utils.StringUtils.applyPagination(supportedOfferings, cmd.getStartIndex(), cmd.getPageSizeVal()); if (wPagination != null) { - final Pair, Integer> listWPagination = new Pair, Integer>(wPagination, supportedOfferings.size()); + final Pair, Integer> listWPagination = new Pair<>(wPagination, supportedOfferings.size()); return listWPagination; } - return new Pair, Integer>(supportedOfferings, supportedOfferings.size()); + return new Pair<>(supportedOfferings, supportedOfferings.size()); } else { final List wPagination = com.cloud.utils.StringUtils.applyPagination(offerings, cmd.getStartIndex(), cmd.getPageSizeVal()); if (wPagination != null) { final Pair, Integer> listWPagination = new Pair<>(wPagination, offerings.size()); return listWPagination; } - return new Pair, Integer>(offerings, offerings.size()); + return new Pair<>(offerings, offerings.size()); } } @@ -8131,7 +8153,7 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati final Integer regionId = cmd.getRegionIdId(); final Long rangeId = cmd.getPortableIpRangeId(); - final List ranges = new ArrayList(); + final List ranges = new ArrayList<>(); if (regionId != null) { final Region region = _regionDao.findById(regionId); if (region == null) { @@ -8256,11 +8278,17 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati }; } + + /** + * Returns a string representing the specified configuration's type. + * @param configName name of the configuration. + * @return if the configuration exists, returns its type; if not, returns {@link Configuration.ValueType#String}. + */ @Override public String getConfigurationType(final String configName) { final ConfigurationVO cfg = _configDao.findByName(configName); if (cfg == null) { - logger.warn("Configuration " + configName + " not found"); + logger.warn("Configuration [{}] not found", configName); return Configuration.ValueType.String.name(); } @@ -8268,24 +8296,14 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return Configuration.ValueType.Range.name(); } - Class type = null; - final Config c = Config.getConfig(configName); - if (c == null) { - logger.warn("Configuration " + configName + " no found. Perhaps moved to ConfigDepot"); - final ConfigKey configKey = _configDepot.get(configName); - if (configKey == null) { - logger.warn("Couldn't find configuration " + configName + " in ConfigDepot too."); - return Configuration.ValueType.String.name(); - } - type = configKey.type(); - } else { - type = c.getType(); - } - - return getInputType(type, cfg); + Class type = getConfigurationTypeWrapperClass(configName); + return parseConfigurationTypeIntoString(type, cfg); } - private String getInputType(Class type, ConfigurationVO cfg) { + /** + * Parses a configuration type's wrapper class into its string representation. + */ + protected String parseConfigurationTypeIntoString(Class type, ConfigurationVO cfg) { if (type == null) { return Configuration.ValueType.String.name(); } @@ -8332,13 +8350,12 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati groupName = configGroup.getName(); } - return new Pair(groupName, subGroupName); + return new Pair<>(groupName, subGroupName); } @Override public List getConfigurationSubGroups(final Long groupId) { - List configSubGroups = _configSubGroupDao.findByGroup(groupId); - return configSubGroups; + return _configSubGroupDao.findByGroup(groupId); } static class ParamCountPair { @@ -8364,10 +8381,6 @@ public class ConfigurationManagerImpl extends ManagerBase implements Configurati return paramCount; } - public void setParamCount(int paramCount) { - this.paramCount = paramCount; - } - public String getScope() { return scope; } diff --git a/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java b/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java index 91e4fddb69c..450f08c46e3 100644 --- a/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkMigrationManagerImpl.java @@ -301,7 +301,7 @@ public class NetworkMigrationManagerImpl implements NetworkMigrationManager { copyOfVpc = _vpcService.createVpc(vpc.getZoneId(), vpcOfferingId, vpc.getAccountId(), vpc.getName(), vpc.getDisplayText(), vpc.getCidr(), vpc.getNetworkDomain(), vpc.getIp4Dns1(), vpc.getIp4Dns2(), - vpc.getIp6Dns1(), vpc.getIp6Dns2(), vpc.isDisplay(), vpc.getPublicMtu(), null, null, null); + vpc.getIp6Dns1(), vpc.getIp6Dns2(), vpc.isDisplay(), vpc.getPublicMtu(), null, null, null, vpc.useRouterIpAsResolver()); copyOfVpcId = copyOfVpc.getId(); //on resume of migration the uuid will be swapped already. So the copy will have the value of the original vpcid. diff --git a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java index 4c4f8bd30bd..ea1bc5c7510 100644 --- a/server/src/main/java/com/cloud/network/NetworkServiceImpl.java +++ b/server/src/main/java/com/cloud/network/NetworkServiceImpl.java @@ -40,6 +40,7 @@ import java.util.UUID; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.dao.ASNumberDao; import org.apache.cloudstack.acl.ControlledEntity.ACLType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.alert.AlertService; @@ -421,6 +422,8 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C RoutedIpv4Manager routedIpv4Manager; @Inject private BGPService bgpService; + @Inject + private ASNumberDao asNumberDao; List internalLoadBalancerElementServices = new ArrayList<>(); Map internalLoadBalancerElementServiceMap = new HashMap<>(); @@ -6284,6 +6287,27 @@ public class NetworkServiceImpl extends ManagerBase implements NetworkService, C return new ArrayList<>(this.internalLoadBalancerElementServiceMap.values()); } + @Override + public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException { + DomainRouterVO router = routerDao.findById(virtualRouterId); + if (router == null) { + String err = String.format("Cannot find VR with ID %s", virtualRouterId); + logger.error(err); + throw new CloudRuntimeException(err); + } + Commands commands = new Commands(Command.OnError.Stop); + commandSetupHelper.createHandleCksIsoCommand(router, mount, commands); + if (!networkHelper.sendCommandsToRouter(router, commands)) { + throw new CloudRuntimeException(String.format("Unable to send commands to virtual router: %s", router.getHostId())); + } + Answer answer = commands.getAnswer("handleCksIso"); + if (answer == null || !answer.getResult()) { + logger.error(String.format("Could not handle the CKS ISO properly: %s", answer.getDetails())); + return false; + } + return true; + } + /** * Retrieves the active quarantine for the given public IP address. It can find by the ID of the quarantine or the address of the public IP. * @throws CloudRuntimeException if it does not find an active quarantine for the given public IP. diff --git a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java index 18ce55aa328..5a53c1016cf 100644 --- a/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java +++ b/server/src/main/java/com/cloud/network/router/CommandSetupHelper.java @@ -28,6 +28,7 @@ import java.util.Set; import javax.inject.Inject; +import com.cloud.agent.api.HandleCksIsoCommand; import com.cloud.network.rules.PortForwardingRuleVO; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; @@ -1425,6 +1426,13 @@ public class CommandSetupHelper { } } + public void createHandleCksIsoCommand(final VirtualRouter router, final boolean mount, Commands cmds) { + HandleCksIsoCommand command = new HandleCksIsoCommand(mount); + command.setAccessDetail(NetworkElementCommand.ROUTER_IP, _routerControlHelper.getRouterControlIp(router.getId())); + command.setAccessDetail(NetworkElementCommand.ROUTER_NAME, router.getInstanceName()); + cmds.addCommand("handleCksIso", command); + } + public void createBgpPeersCommands(final List bgpPeers, final VirtualRouter router, final Commands cmds, final Network network) { List bgpPeerTOs = new ArrayList<>(); diff --git a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index e171b68399b..27f04234c33 100644 --- a/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/main/java/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -2072,6 +2072,7 @@ Configurable, StateListener bgpPeerIds) throws ResourceAllocationException { + final Integer cidrSize, final Long asNumber, final List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException { final Account caller = CallContext.current().getCallingAccount(); final Account owner = _accountMgr.getAccount(vpcOwnerId); @@ -1247,6 +1247,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis vpcOff.isRedundantRouter(), ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2); vpc.setPublicMtu(publicMtu); vpc.setDisplay(Boolean.TRUE.equals(displayVpc)); + vpc.setUseRouterIpResolver(Boolean.TRUE.equals(useVrIpResolver)); if (vpc.getCidr() == null && cidrSize != null) { // Allocate a CIDR for VPC @@ -1305,7 +1306,7 @@ public class VpcManagerImpl extends ManagerBase implements VpcManager, VpcProvis List bgpPeerIds = (cmd instanceof CreateVPCCmdByAdmin) ? ((CreateVPCCmdByAdmin)cmd).getBgpPeerIds() : null; Vpc vpc = createVpc(cmd.getZoneId(), cmd.getVpcOffering(), cmd.getEntityOwnerId(), cmd.getVpcName(), cmd.getDisplayText(), cmd.getCidr(), cmd.getNetworkDomain(), cmd.getIp4Dns1(), cmd.getIp4Dns2(), cmd.getIp6Dns1(), - cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu(), cmd.getCidrSize(), cmd.getAsNumber(), bgpPeerIds); + cmd.getIp6Dns2(), cmd.isDisplay(), cmd.getPublicMtu(), cmd.getCidrSize(), cmd.getAsNumber(), bgpPeerIds, cmd.getUseVrIpResolver()); String sourceNatIP = cmd.getSourceNatIP(); boolean forNsx = isVpcForNsx(vpc); diff --git a/server/src/main/java/com/cloud/resourceicon/ResourceIconManagerImpl.java b/server/src/main/java/com/cloud/resourceicon/ResourceIconManagerImpl.java index 6c286edd00d..2a7881abb70 100644 --- a/server/src/main/java/com/cloud/resourceicon/ResourceIconManagerImpl.java +++ b/server/src/main/java/com/cloud/resourceicon/ResourceIconManagerImpl.java @@ -16,8 +16,11 @@ // under the License. package com.cloud.resourceicon; +import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import javax.inject.Inject; @@ -247,4 +250,18 @@ public class ResourceIconManagerImpl extends ManagerBase implements ResourceIcon sc.setParameters("resourceType", resourceType); return resourceIconDao.search(sc, null); } + + + @Override + public Map getByResourceTypeAndIds(ResourceTag.ResourceObjectType resourceType, Collection resourceIds) { + List icons = resourceIconDao.listByResourceTypeAndIds(resourceType, resourceIds); + return icons.stream().collect(Collectors.toMap(ResourceIconVO::getResourceId, Function.identity())); + } + + + @Override + public Map getByResourceTypeAndUuids(ResourceTag.ResourceObjectType resourceType, Collection resourceUuids) { + List icons = resourceIconDao.listByResourceTypeAndUuids(resourceType, resourceUuids); + return icons.stream().collect(Collectors.toMap(ResourceIconVO::getResourceUuid, Function.identity())); + } } diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index de0aa286a97..9ac4599f30e 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -44,6 +44,8 @@ import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.api.query.dao.ManagementServerJoinDao; +import com.cloud.api.query.vo.ManagementServerJoinVO; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker; import org.apache.cloudstack.affinity.AffinityGroupProcessor; @@ -89,12 +91,15 @@ import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmd; import org.apache.cloudstack.api.command.admin.domain.ListDomainsCmdByAdmin; import org.apache.cloudstack.api.command.admin.domain.MoveDomainCmd; import org.apache.cloudstack.api.command.admin.domain.UpdateDomainCmd; +import org.apache.cloudstack.api.command.admin.guest.AddGuestOsCategoryCmd; import org.apache.cloudstack.api.command.admin.guest.AddGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.AddGuestOsMappingCmd; +import org.apache.cloudstack.api.command.admin.guest.DeleteGuestOsCategoryCmd; import org.apache.cloudstack.api.command.admin.guest.GetHypervisorGuestOsNamesCmd; import org.apache.cloudstack.api.command.admin.guest.ListGuestOsMappingCmd; import org.apache.cloudstack.api.command.admin.guest.RemoveGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.RemoveGuestOsMappingCmd; +import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsCategoryCmd; import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsCmd; import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsMappingCmd; import org.apache.cloudstack.api.command.admin.host.AddHostCmd; @@ -125,6 +130,7 @@ import org.apache.cloudstack.api.command.admin.iso.ListIsosCmdByAdmin; import org.apache.cloudstack.api.command.admin.iso.RegisterIsoCmdByAdmin; import org.apache.cloudstack.api.command.admin.loadbalancer.ListLoadBalancerRuleInstancesCmdByAdmin; import org.apache.cloudstack.api.command.admin.management.ListMgmtsCmd; +import org.apache.cloudstack.api.command.admin.management.RemoveManagementServerCmd; import org.apache.cloudstack.api.command.admin.network.AddNetworkDeviceCmd; import org.apache.cloudstack.api.command.admin.network.AddNetworkServiceProviderCmd; import org.apache.cloudstack.api.command.admin.network.CreateManagementNetworkIpRangeCmd; @@ -521,9 +527,13 @@ import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; import org.apache.cloudstack.api.command.user.template.RegisterTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; +import org.apache.cloudstack.api.command.user.userdata.BaseRegisterUserDataCmd; +import org.apache.cloudstack.api.command.user.userdata.DeleteCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; import org.apache.cloudstack.api.command.user.userdata.LinkUserDataToTemplateCmd; +import org.apache.cloudstack.api.command.user.userdata.ListCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; +import org.apache.cloudstack.api.command.user.userdata.RegisterCniConfigurationCmd; import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; import org.apache.cloudstack.api.command.user.vm.AddIpToVmNicCmd; import org.apache.cloudstack.api.command.user.vm.AddNicToVMCmd; @@ -627,6 +637,7 @@ import org.apache.cloudstack.framework.config.impl.ConfigurationSubGroupVO; import org.apache.cloudstack.framework.config.impl.ConfigurationVO; import org.apache.cloudstack.framework.security.keystore.KeystoreManager; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.GuestOsDetailsDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; @@ -643,6 +654,8 @@ import org.apache.cloudstack.utils.identity.ManagementServerNode; import org.apache.cloudstack.vm.lease.VMLeaseManager; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import com.cloud.agent.AgentManager; @@ -1015,6 +1028,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe UserDataManager userDataManager; @Inject StoragePoolTagsDao storagePoolTagsDao; + @Inject + protected ManagementServerJoinDao managementServerJoinDao; @Inject private PublicIpQuarantineDao publicIpQuarantineDao; @@ -2752,29 +2767,114 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Override public Pair, Integer> listGuestOSCategoriesByCriteria(final ListGuestOsCategoriesCmd cmd) { - final Filter searchFilter = new Filter(GuestOSCategoryVO.class, "id", true, cmd.getStartIndex(), cmd.getPageSizeVal()); + final Filter searchFilter = new Filter(GuestOSCategoryVO.class, "sortKey", true, + cmd.getStartIndex(), cmd.getPageSizeVal()); + searchFilter.addOrderBy(GuestOSCategoryVO.class, "id", true); final Long id = cmd.getId(); final String name = cmd.getName(); final String keyword = cmd.getKeyword(); + final Boolean featured = cmd.isFeatured(); + final Boolean isIso = cmd.isIso(); + final Boolean isVnf = cmd.isVnf(); + final Long zoneId = cmd.getZoneId(); + final CPU.CPUArch arch = cmd.getArch(); - final SearchCriteria sc = _guestOSCategoryDao.createSearchCriteria(); - + final SearchBuilder sb = _guestOSCategoryDao.createSearchBuilder(); + sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("featured", sb.entity().isFeatured(), SearchCriteria.Op.EQ); + if (ObjectUtils.anyNotNull(zoneId, arch, isIso, isVnf)) { + final SearchBuilder guestOsSearch = _guestOSDao.createSearchBuilder(); + guestOsSearch.and("ids", guestOsSearch.entity().getId(), SearchCriteria.Op.IN); + sb.join("guestOsSearch", guestOsSearch, guestOsSearch.entity().getCategoryId(), sb.entity().getId(), + JoinType.INNER); + guestOsSearch.done(); + sb.groupBy(sb.entity().getId()); + } + sb.done(); + SearchCriteria sc = sb.create(); if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); + sc.setParameters("id", id); } - if (name != null) { - sc.addAnd("name", SearchCriteria.Op.LIKE, "%" + name + "%"); + sc.setParameters("name", "%" + name + "%"); } - if (keyword != null) { - sc.addAnd("name", SearchCriteria.Op.LIKE, "%" + keyword + "%"); + sc.setParameters("name", "%" + keyword + "%"); + } + if (featured != null) { + sc.setParameters("featured", featured); + } + if (ObjectUtils.anyNotNull(zoneId, arch, isIso, isVnf)) { + List guestOsIds = templateDao.listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf); + if (CollectionUtils.isEmpty(guestOsIds)) { + return new Pair<>(Collections.emptyList(), 0); + } + sc.setJoinParameters("guestOsSearch", "ids", guestOsIds.toArray()); } - final Pair, Integer> result = _guestOSCategoryDao.searchAndCount(sc, searchFilter); return new Pair<>(result.first(), result.second()); } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_GUEST_OS_CATEGORY_ADD, eventDescription = "adding OS category") + public GuestOsCategory addGuestOsCategory(AddGuestOsCategoryCmd cmd) { + final String name = cmd.getName(); + final boolean featured = cmd.isFeatured(); + final GuestOSCategoryVO guestOSCategory = new GuestOSCategoryVO(name, featured); + GuestOsCategory guestOsCategory = _guestOSCategoryDao.persist(guestOSCategory); + CallContext.current().setEventResourceId(guestOsCategory.getId()); + CallContext.current().setEventResourceType(ApiCommandResourceType.GuestOsCategory); + return guestOSCategory; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_GUEST_OS_CATEGORY_UPDATE, eventDescription = "updating OS category") + public GuestOsCategory updateGuestOsCategory(UpdateGuestOsCategoryCmd cmd) { + final long id = cmd.getId(); + final String name = cmd.getName(); + final Boolean featured = cmd.isFeatured(); + Integer sortKey = cmd.getSortKey(); + final GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(id); + if (guestOSCategory == null) { + throw new InvalidParameterValueException("Invalid OS category ID specified"); + } + if (ObjectUtils.allNull(name, featured, sortKey)) { + return guestOSCategory; + } + if (StringUtils.isNotBlank(name)) { + guestOSCategory.setName(name); + } + if (featured != null) { + guestOSCategory.setFeatured(featured); + } + if (sortKey != null) { + guestOSCategory.setSortKey(sortKey); + } + if (!_guestOSCategoryDao.update(id, guestOSCategory)) { + return null; + } + return guestOSCategory; + } + + @Override + @ActionEvent(eventType = EventTypes.EVENT_GUEST_OS_CATEGORY_DELETE, eventDescription = "deleting OS category") + public boolean deleteGuestOsCategory(DeleteGuestOsCategoryCmd cmd) { + final long id = cmd.getId(); + final GuestOSCategoryVO guestOSCategory = _guestOSCategoryDao.findById(id); + if (guestOSCategory == null) { + throw new InvalidParameterValueException("Invalid OS category ID specified"); + } + List guestOses = _guestOSDao.listIdsByCategoryId(id); + if (!guestOses.isEmpty()) { + throw new InvalidParameterValueException(String.format( + "Unable to delete the OS category. %d guest OS exist for it.", guestOses.size())); + } + return _guestOSCategoryDao.remove(id); + } + @Override public Pair, Integer> listGuestOSMappingByCriteria(final ListGuestOsMappingCmd cmd) { final String guestOsId = "guestOsId"; @@ -2992,6 +3092,10 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe public GuestOS updateGuestOs(final UpdateGuestOsCmd cmd) { final Long id = cmd.getId(); final String displayName = cmd.getOsDisplayName(); + final Long osCategoryId = cmd.getOsCategoryId(); + final Boolean display = cmd.getForDisplay(); + final Map details = cmd.getDetails(); + boolean updateNeeded = false; //check if guest OS exists final GuestOS guestOsHandle = ApiDBUtils.findGuestOSById(id); @@ -2999,27 +3103,46 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe throw new InvalidParameterValueException("Guest OS not found. Please specify a valid ID for the Guest OS"); } - if (!guestOsHandle.getIsUserDefined()) { + //Check if update is needed + if (StringUtils.isNotBlank(displayName) && !displayName.equals(guestOsHandle.getDisplayName())) { + //Check if another Guest OS by same name exists + final GuestOS duplicate = ApiDBUtils.findGuestOSByDisplayName(displayName); + if (duplicate != null) { + throw new InvalidParameterValueException("The specified Guest OS name : " + displayName + " already exists. Please specify a unique guest OS name"); + } + updateNeeded = true; + } + + if (osCategoryId != null) { + if (_guestOSCategoryDao.findById(osCategoryId) == null) { + throw new InvalidParameterValueException("Invalid OS category ID specified"); + } + updateNeeded = true; + } + + if (!guestOsHandle.getIsUserDefined() && (StringUtils.isNotBlank(displayName) || MapUtils.isNotEmpty(details) + || display != null)) { throw new InvalidParameterValueException("Unable to modify system defined guest OS"); } - persistGuestOsDetails(cmd.getDetails(), id); + if (MapUtils.isNotEmpty(details)) { + persistGuestOsDetails(details, id); + } - //Check if update is needed - if (displayName.equals(guestOsHandle.getDisplayName())) { + if (!updateNeeded) { return guestOsHandle; } - //Check if another Guest OS by same name exists - final GuestOS duplicate = ApiDBUtils.findGuestOSByDisplayName(displayName); - if (duplicate != null) { - throw new InvalidParameterValueException("The specified Guest OS name : " + displayName + " already exists. Please specify a unique guest OS name"); - } final GuestOSVO guestOs = _guestOSDao.createForUpdate(id); - guestOs.setDisplayName(displayName); + if (StringUtils.isNotBlank(displayName)) { + guestOs.setDisplayName(displayName); + } if (cmd.getForDisplay() != null) { guestOs.setDisplay(cmd.getForDisplay()); } + if (osCategoryId != null) { + guestOs.setCategoryId(osCategoryId); + } if (_guestOSDao.update(id, guestOs)) { return _guestOSDao.findById(id); } else { @@ -3722,6 +3845,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(ListPortForwardingRulesCmd.class); cmdList.add(UpdatePortForwardingRuleCmd.class); cmdList.add(ListGuestOsCategoriesCmd.class); + cmdList.add(AddGuestOsCategoryCmd.class); + cmdList.add(UpdateGuestOsCategoryCmd.class); + cmdList.add(DeleteGuestOsCategoryCmd.class); cmdList.add(ListGuestOsCmd.class); cmdList.add(ListGuestOsMappingCmd.class); cmdList.add(AddGuestOsCmd.class); @@ -4041,6 +4167,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(ListTemplateDirectDownloadCertificatesCmd.class); cmdList.add(ProvisionTemplateDirectDownloadCertificateCmd.class); cmdList.add(ListMgmtsCmd.class); + cmdList.add(RemoveManagementServerCmd.class); cmdList.add(GetUploadParamsForIsoCmd.class); cmdList.add(GetRouterHealthCheckResultsCmd.class); cmdList.add(StartRollingMaintenanceCmd.class); @@ -4076,6 +4203,9 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(DeleteUserDataCmd.class); cmdList.add(ListUserDataCmd.class); cmdList.add(LinkUserDataToTemplateCmd.class); + cmdList.add(RegisterCniConfigurationCmd.class); + cmdList.add(ListCniConfigurationCmd.class); + cmdList.add(DeleteCniConfigurationCmd.class); //object store APIs cmdList.add(AddObjectStoragePoolCmd.class); @@ -4888,7 +5018,13 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe } @Override - public Pair, Integer> listUserDatas(final ListUserDataCmd cmd) { + @ActionEvent(eventType = EventTypes.EVENT_DELETE_CNI_CONFIG, eventDescription = "CNI Configuration deletion") + public boolean deleteCniConfiguration(DeleteCniConfigurationCmd cmd) { + return deleteUserData(cmd); + } + + @Override + public Pair, Integer> listUserDatas(final ListUserDataCmd cmd, final boolean forCks) { final Long id = cmd.getId(); final String name = cmd.getName(); final String keyword = cmd.getKeyword(); @@ -4908,6 +5044,8 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sb.and("id", sb.entity().getId(), SearchCriteria.Op.EQ); sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); sb.and("keyword", sb.entity().getName(), SearchCriteria.Op.LIKE); + sb.and("name", sb.entity().getName(), SearchCriteria.Op.EQ); + sb.and("forCks", sb.entity().isForCks(), SearchCriteria.Op.EQ); final SearchCriteria sc = sb.create(); _accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccounts, listProjectResourcesCriteria); @@ -4923,24 +5061,41 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe sc.setParameters("keyword", "%" + keyword + "%"); } + sc.setParameters("forCks", forCks); + final Pair, Integer> result = userDataDao.searchAndCount(sc, searchFilter); return new Pair<>(result.first(), result.second()); } + @Override + @ActionEvent(eventType = EventTypes.EVENT_REGISTER_CNI_CONFIG, eventDescription = "registering CNI configuration", async = true) + public UserData registerCniConfiguration(RegisterCniConfigurationCmd cmd) { + final Account owner = getOwner(cmd); + checkForUserDataByName(cmd, owner); + final String name = cmd.getName(); + + String userdata = cmd.getCniConfig(); + final String params = cmd.getParams(); + + userdata = userDataManager.validateUserData(userdata, cmd.getHttpMethod()); + + return createAndSaveUserData(name, userdata, params, owner, true); + } + @Override @ActionEvent(eventType = EventTypes.EVENT_REGISTER_USER_DATA, eventDescription = "registering userdata", async = true) public UserData registerUserData(final RegisterUserDataCmd cmd) { final Account owner = getOwner(cmd); checkForUserDataByName(cmd, owner); - checkForUserData(cmd, owner); - final String name = cmd.getName(); + String userdata = cmd.getUserData(); + checkForUserData(cmd, owner); final String params = cmd.getParams(); userdata = userDataManager.validateUserData(userdata, cmd.getHttpMethod()); - return createAndSaveUserData(name, userdata, params, owner); + return createAndSaveUserData(name, userdata, params, owner, false); } /** @@ -4960,7 +5115,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe * @param owner * @throws InvalidParameterValueException */ - private void checkForUserDataByName(final RegisterUserDataCmd cmd, final Account owner) throws InvalidParameterValueException { + private void checkForUserDataByName(final BaseRegisterUserDataCmd cmd, final Account owner) throws InvalidParameterValueException { final UserDataVO userData = userDataDao.findByName(owner.getAccountId(), owner.getDomainId(), cmd.getName()); if (userData != null) { throw new InvalidParameterValueException(String.format("A userdata with name %s already exists for this account.", cmd.getName())); @@ -5029,7 +5184,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe * @param cmd * @return Account */ - protected Account getOwner(final RegisterUserDataCmd cmd) { + protected Account getOwner(final BaseRegisterUserDataCmd cmd) { final Account caller = getCaller(); return _accountMgr.finalizeOwner(caller, cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId()); } @@ -5042,7 +5197,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe return caller; } - private SSHKeyPair createAndSaveSSHKeyPair(final String name, final String fingerprint, final String publicKey, final String privateKey, final Account owner) { + private SSHKeyPair createAndSaveSSHKeyPair(final String name, final String fingerprint, final String publicKey, final String privateKey, final Account owner) { final SSHKeyPairVO newPair = new SSHKeyPairVO(); newPair.setAccountId(owner.getAccountId()); @@ -5057,7 +5212,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe return newPair; } - private UserData createAndSaveUserData(final String name, final String userdata, final String params, final Account owner) { + private UserData createAndSaveUserData(final String name, final String userdata, final String params, final Account owner, final boolean isForCks) { final UserDataVO userDataVO = new UserDataVO(); userDataVO.setAccountId(owner.getAccountId()); @@ -5065,6 +5220,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe userDataVO.setName(name); userDataVO.setUserData(userdata); userDataVO.setParams(params); + userDataVO.setForCks(isForCks); userDataDao.persist(userDataVO); @@ -5552,4 +5708,24 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe _lockControllerListener = lockControllerListener; } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_MANAGEMENT_SERVER_REMOVE, eventDescription = "removing Management Server") + public boolean removeManagementServer(RemoveManagementServerCmd cmd) { + final Long id = cmd.getId(); + ManagementServerJoinVO managementServer = managementServerJoinDao.findById(id); + + if (managementServer == null) { + throw new InvalidParameterValueException(String.format("Unable to find a Management Server with ID equal to [%s].", managementServer.getUuid())); + } + + if (!ManagementServerHost.State.Down.equals(managementServer.getState())) { + throw new InvalidParameterValueException(String.format("Unable to remove Management Server with ID [%s]. It can only be removed when it is in the [%s] state, however currently it is in the [%s] state.", managementServer.getUuid(), ManagementServerHost.State.Down.name(), managementServer.getState().name())); + } + + managementServer.setRemoved(new Date()); + return managementServerJoinDao.update(id, managementServer); + + } + } diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 5de7ade696a..f144745fc5c 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -265,6 +265,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionCallbackWithExceptionNoReturn; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; @@ -1834,22 +1835,27 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (exceptionOccurred.get()) { return null; } - HostVO host = _hostDao.findById(hostId); - try { - connectHostToSharedPool(host, primaryStore.getId()); - poolHostIds.add(hostId); - } catch (Exception e) { - if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) { - exceptionOccurred.set(true); - throw e; + Transaction.execute(new TransactionCallbackWithExceptionNoReturn() { + @Override + public void doInTransactionWithoutResult(TransactionStatus status) throws Exception { + HostVO host = _hostDao.findById(hostId); + try { + connectHostToSharedPool(host, primaryStore.getId()); + poolHostIds.add(hostId); + } catch (Exception e) { + if (handleExceptionsPartially && e.getCause() instanceof StorageConflictException) { + exceptionOccurred.set(true); + throw e; + } + logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e); + String reason = getStoragePoolMountFailureReason(e.getMessage()); + if (handleExceptionsPartially && reason != null) { + exceptionOccurred.set(true); + throw new CloudRuntimeException(reason); + } + } } - logger.warn("Unable to establish a connection between {} and {}", host, primaryStore, e); - String reason = getStoragePoolMountFailureReason(e.getMessage()); - if (handleExceptionsPartially && reason != null) { - exceptionOccurred.set(true); - throw new CloudRuntimeException(reason); - } - } + }); return null; })); } diff --git a/server/src/main/java/com/cloud/storage/TemplateProfile.java b/server/src/main/java/com/cloud/storage/TemplateProfile.java index 49fc6836d73..c2a3c506221 100644 --- a/server/src/main/java/com/cloud/storage/TemplateProfile.java +++ b/server/src/main/java/com/cloud/storage/TemplateProfile.java @@ -55,6 +55,7 @@ public class TemplateProfile { TemplateType templateType; Boolean directDownload; Boolean deployAsIs; + Boolean forCks; Long size; public TemplateProfile(Long templateId, Long userId, String name, String displayText, CPU.CPUArch arch, Integer bits, Boolean passwordEnabled, Boolean requiresHvm, String url, @@ -342,6 +343,14 @@ public class TemplateProfile { return this.deployAsIs; } + public Boolean isForCks() { + return forCks; + } + + public void setForCks(Boolean forCks) { + this.forCks = forCks; + } + public CPU.CPUArch getArch() { return arch; } diff --git a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java index 2048ee4cfc9..61a93564255 100644 --- a/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/VolumeApiServiceImpl.java @@ -2435,6 +2435,9 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new InvalidParameterValueException("Going from existing size of " + currentSize + " to size of " + newSize + " would shrink the volume." + "Need to sign off by supplying the shrinkok parameter with value of true."); } + if (ApiDBUtils.getHypervisorTypeFromFormat(volume.getDataCenterId(), volume.getFormat()) == HypervisorType.XenServer) { + throw new InvalidParameterValueException("Shrink volume is not supported for the XenServer hypervisor."); + } } } /* Check resource limit for this account */ diff --git a/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java b/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java index d11edce14c2..769aa3dc1f2 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java +++ b/server/src/main/java/com/cloud/storage/upload/params/TemplateUploadParams.java @@ -30,10 +30,10 @@ public class TemplateUploadParams extends UploadParamsBase { Long zoneId, Hypervisor.HypervisorType hypervisorType, String chksum, String templateTag, long templateOwnerId, Map details, Boolean sshkeyEnabled, - Boolean isDynamicallyScalable, Boolean isRoutingType, boolean deployAsIs) { + Boolean isDynamicallyScalable, Boolean isRoutingType, boolean deployAsIs, boolean forCks) { super(userId, name, displayText, arch, bits, passwordEnabled, requiresHVM, isPublic, featured, isExtractable, format, guestOSId, zoneId, hypervisorType, chksum, templateTag, templateOwnerId, details, - sshkeyEnabled, isDynamicallyScalable, isRoutingType, deployAsIs); + sshkeyEnabled, isDynamicallyScalable, isRoutingType, deployAsIs, forCks); setBootable(true); } } diff --git a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java index 3bf3e77fe1d..c3499d75c3b 100644 --- a/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java +++ b/server/src/main/java/com/cloud/storage/upload/params/UploadParamsBase.java @@ -46,6 +46,7 @@ public abstract class UploadParamsBase implements UploadParams { private boolean isDynamicallyScalable; private boolean isRoutingType; private boolean deployAsIs; + private boolean forCks; private CPU.CPUArch arch; UploadParamsBase(long userId, String name, String displayText, CPU.CPUArch arch, @@ -55,7 +56,7 @@ public abstract class UploadParamsBase implements UploadParams { Long zoneId, Hypervisor.HypervisorType hypervisorType, String checksum, String templateTag, long templateOwnerId, Map details, boolean sshkeyEnabled, - boolean isDynamicallyScalable, boolean isRoutingType, boolean deployAsIs) { + boolean isDynamicallyScalable, boolean isRoutingType, boolean deployAsIs, boolean forCks) { this.userId = userId; this.name = name; this.displayText = displayText; @@ -232,6 +233,10 @@ public abstract class UploadParamsBase implements UploadParams { this.bootable = bootable; } + void setForCks(boolean forCks) { + this.forCks = forCks; + } + void setBits(Integer bits) { this.bits = bits; } diff --git a/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java b/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java index c02f4136863..e6d0b737bbe 100644 --- a/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java +++ b/server/src/main/java/com/cloud/tags/ResourceManagerUtilImpl.java @@ -58,6 +58,7 @@ import com.cloud.server.ResourceManagerUtil; import com.cloud.server.ResourceTag; import com.cloud.service.ServiceOfferingVO; import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.GuestOsCategory; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.VMTemplateVO; @@ -117,6 +118,7 @@ public class ResourceManagerUtilImpl implements ResourceManagerUtil { s_typeMap.put(ResourceTag.ResourceObjectType.NetworkOffering, NetworkOfferingVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.VpcOffering, VpcOfferingVO.class); s_typeMap.put(ResourceTag.ResourceObjectType.Domain, DomainVO.class); + s_typeMap.put(ResourceTag.ResourceObjectType.GuestOsCategory, GuestOsCategory.class); } @Inject diff --git a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java index f7eb654141d..d50d9e71df3 100644 --- a/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/HypervisorTemplateAdapter.java @@ -246,6 +246,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { Long templateSize = performDirectDownloadUrlValidation(cmd.getFormat(), hypervisor, url, cmd.getZoneIds(), followRedirects); profile.setSize(templateSize); + profile.setForCks(cmd.isForCks()); } profile.setUrl(url); // Check that the resource limit for secondary storage won't be exceeded diff --git a/server/src/main/java/com/cloud/template/TemplateAdapter.java b/server/src/main/java/com/cloud/template/TemplateAdapter.java index 27ff563655d..32a8db515aa 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapter.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapter.java @@ -79,6 +79,6 @@ public interface TemplateAdapter extends Adapter { TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, CPU.CPUArch arch, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String chksum, Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshKeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, - TemplateType templateType, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException; + TemplateType templateType, boolean directDownload, boolean deployAsIs, boolean forCks) throws ResourceAllocationException; } diff --git a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java index 5b7185b94c5..20affe4f6f9 100644 --- a/server/src/main/java/com/cloud/template/TemplateAdapterBase.java +++ b/server/src/main/java/com/cloud/template/TemplateAdapterBase.java @@ -136,14 +136,14 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneId, HypervisorType hypervisorType, String accountName, Long domainId, String chksum, Boolean bootable, Map details, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException { return prepare(isIso, userId, name, displayText, arch, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, format, guestOSId, zoneId, - hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload, deployAsIs); + hypervisorType, chksum, bootable, null, null, details, false, null, false, TemplateType.USER, directDownload, deployAsIs, false); } @Override public TemplateProfile prepare(boolean isIso, long userId, String name, String displayText, CPU.CPUArch arch, Integer bits, Boolean passwordEnabled, Boolean requiresHVM, String url, Boolean isPublic, Boolean featured, Boolean isExtractable, String format, Long guestOSId, List zoneIdList, HypervisorType hypervisorType, String chksum, Boolean bootable, String templateTag, Account templateOwner, Map details, Boolean sshkeyEnabled, String imageStoreUuid, Boolean isDynamicallyScalable, - TemplateType templateType, boolean directDownload, boolean deployAsIs) throws ResourceAllocationException { + TemplateType templateType, boolean directDownload, boolean deployAsIs, boolean forCks) throws ResourceAllocationException { //Long accountId = null; // parameters verification @@ -268,9 +268,11 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat Long id = _tmpltDao.getNextInSequence(Long.class, "id"); CallContext.current().setEventDetails("Id: " + id + " name: " + name); - return new TemplateProfile(id, userId, name, displayText, arch, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList, + TemplateProfile profile = new TemplateProfile(id, userId, name, displayText, arch, bits, passwordEnabled, requiresHVM, url, isPublic, featured, isExtractable, imgfmt, guestOSId, zoneIdList, hypervisorType, templateOwner.getAccountName(), templateOwner.getDomainId(), templateOwner.getAccountId(), chksum, bootable, templateTag, details, sshkeyEnabled, null, isDynamicallyScalable, templateType, directDownload, deployAsIs); + profile.setForCks(forCks); + return profile; } @@ -315,7 +317,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat return prepare(false, CallContext.current().getCallingUserId(), cmd.getTemplateName(), cmd.getDisplayText(), cmd.getArch(), cmd.getBits(), cmd.isPasswordEnabled(), cmd.getRequiresHvm(), cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), cmd.getFormat(), cmd.getOsTypeId(), zoneId, hypervisorType, cmd.getChecksum(), true, cmd.getTemplateTag(), owner, details, cmd.isSshKeyEnabled(), null, cmd.isDynamicallyScalable(), templateType, - cmd.isDirectDownload(), cmd.isDeployAsIs()); + cmd.isDirectDownload(), cmd.isDeployAsIs(), cmd.isForCks()); } @@ -348,7 +350,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat params.isExtractable(), params.getFormat(), params.getGuestOSId(), zoneList, params.getHypervisorType(), params.getChecksum(), params.isBootable(), params.getTemplateTag(), owner, params.getDetails(), params.isSshKeyEnabled(), params.getImageStoreUuid(), - params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload(), params.isDeployAsIs()); + params.isDynamicallyScalable(), params.isRoutingType() ? TemplateType.ROUTING : TemplateType.USER, params.isDirectDownload(), params.isDeployAsIs(), false); } private Long getDefaultDeployAsIsGuestOsId() { @@ -369,7 +371,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat BooleanUtils.toBoolean(cmd.isFeatured()), BooleanUtils.toBoolean(cmd.isExtractable()), cmd.getFormat(), osTypeId, cmd.getZoneId(), HypervisorType.getType(cmd.getHypervisor()), cmd.getChecksum(), cmd.getTemplateTag(), cmd.getEntityOwnerId(), cmd.getDetails(), BooleanUtils.toBoolean(cmd.isSshKeyEnabled()), - BooleanUtils.toBoolean(cmd.isDynamicallyScalable()), BooleanUtils.toBoolean(cmd.isRoutingType()), cmd.isDeployAsIs()); + BooleanUtils.toBoolean(cmd.isDynamicallyScalable()), BooleanUtils.toBoolean(cmd.isRoutingType()), cmd.isDeployAsIs(), cmd.isForCks()); return prepareUploadParamsInternal(params); } @@ -400,7 +402,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat return prepare(true, CallContext.current().getCallingUserId(), cmd.getIsoName(), cmd.getDisplayText(), cmd.getArch(), 64, cmd.isPasswordEnabled(), true, cmd.getUrl(), cmd.isPublic(), cmd.isFeatured(), cmd.isExtractable(), ImageFormat.ISO.toString(), cmd.getOsTypeId(), zoneList, HypervisorType.None, cmd.getChecksum(), cmd.isBootable(), null, - owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload(), false); + owner, null, false, cmd.getImageStoreUuid(), cmd.isDynamicallyScalable(), TemplateType.USER, cmd.isDirectDownload(), false, false); } protected VMTemplateVO persistTemplate(TemplateProfile profile, VirtualMachineTemplate.State initialState) { @@ -411,6 +413,7 @@ public abstract class TemplateAdapterBase extends AdapterBase implements Templat profile.getDisplayText(), profile.isPasswordEnabled(), profile.getGuestOsId(), profile.isBootable(), profile.getHypervisorType(), profile.getTemplateTag(), profile.getDetails(), profile.isSshKeyEnabled(), profile.IsDynamicallyScalable(), profile.isDirectDownload(), profile.isDeployAsIs(), profile.getArch()); template.setState(initialState); + template.setForCks(profile.isForCks()); if (profile.isDirectDownload()) { template.setSize(profile.getSize()); diff --git a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java index 874ad120baf..6759144837d 100755 --- a/server/src/main/java/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/main/java/com/cloud/template/TemplateManagerImpl.java @@ -35,6 +35,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import com.cloud.cpu.CPU; +import com.cloud.vm.VirtualMachine; import com.cloud.storage.snapshot.SnapshotManager; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ApiConstants; @@ -1144,35 +1145,33 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_DETACH, eventDescription = "detaching ISO", async = true) - public boolean detachIso(long vmId, boolean forced) { + public boolean detachIso(long vmId, Long isoParamId, Boolean... extraParams) { Account caller = CallContext.current().getCallingAccount(); Long userId = CallContext.current().getCallingUserId(); - // Verify input parameters - UserVmVO vmInstanceCheck = _userVmDao.findById(vmId); - if (vmInstanceCheck == null) { - throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); - } + boolean forced = extraParams != null && extraParams.length > 0 ? extraParams[0] : false; + boolean isVirtualRouter = extraParams != null && extraParams.length > 1 ? extraParams[1] : false; - UserVm userVM = _userVmDao.findById(vmId); - if (userVM == null) { + // Verify input parameters + VirtualMachine virtualMachine = !isVirtualRouter ? _userVmDao.findById(vmId) : _vmInstanceDao.findById(vmId); + if (virtualMachine == null || (isVirtualRouter && virtualMachine.getType() != VirtualMachine.Type.DomainRouter)) { throw new InvalidParameterValueException("Please specify a valid VM."); } - _accountMgr.checkAccess(caller, null, true, userVM); + _accountMgr.checkAccess(caller, null, true, virtualMachine); - Long isoId = userVM.getIsoId(); + Long isoId = !isVirtualRouter ? ((UserVm) virtualMachine).getIsoId() : isoParamId; if (isoId == null) { throw new InvalidParameterValueException("The specified VM has no ISO attached to it."); } - CallContext.current().setEventDetails("Vm Id: " + userVM.getUuid() + " ISO Id: " + isoId); + CallContext.current().setEventDetails("Vm Id: " + virtualMachine.getUuid() + " ISO Id: " + isoId); - State vmState = userVM.getState(); + State vmState = virtualMachine.getState(); if (vmState != State.Running && vmState != State.Stopped) { throw new InvalidParameterValueException("Please specify a VM that is either Stopped or Running."); } - boolean result = attachISOToVM(vmId, userId, isoId, false, forced); // attach=false + boolean result = attachISOToVM(vmId, userId, isoId, false, forced, isVirtualRouter); // attach=false // => detach if (result) { return result; @@ -1183,16 +1182,28 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Override @ActionEvent(eventType = EventTypes.EVENT_ISO_ATTACH, eventDescription = "attaching ISO", async = true) - public boolean attachIso(long isoId, long vmId, boolean forced) { + public boolean attachIso(long isoId, long vmId, Boolean... extraParams) { Account caller = CallContext.current().getCallingAccount(); Long userId = CallContext.current().getCallingUserId(); + boolean forced = extraParams != null && extraParams.length > 0 ? extraParams[0] : false; + boolean isVirtualRouter = extraParams != null && extraParams.length > 1 ? extraParams[1] : false; + // Verify input parameters - UserVmVO vm = _userVmDao.findById(vmId); + VirtualMachine vm = _userVmDao.findById(vmId); if (vm == null) { - throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); + if (isVirtualRouter) { + vm = _vmInstanceDao.findById(vmId); + if (vm == null) { + throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); + } else if (vm.getType() != VirtualMachine.Type.DomainRouter) { + throw new InvalidParameterValueException("Unable to find a virtual router with id " + vmId); + } + } else { + throw new InvalidParameterValueException("Unable to find a virtual machine with id " + vmId); + } } - if (UserVmManager.SHAREDFSVM.equals(vm.getUserVmType())) { + if (vm instanceof UserVm && UserVmManager.SHAREDFSVM.equals(((UserVm) vm).getUserVmType())) { throw new InvalidParameterValueException("Operation not supported on Shared FileSystem Instance"); } @@ -1229,7 +1240,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, if (VMWARE_TOOLS_ISO.equals(iso.getUniqueName()) && vm.getHypervisorType() != Hypervisor.HypervisorType.VMware) { throw new InvalidParameterValueException("Cannot attach VMware tools drivers to incompatible hypervisor " + vm.getHypervisorType()); } - boolean result = attachISOToVM(vmId, userId, isoId, true, forced); + boolean result = attachISOToVM(vmId, userId, isoId, true, forced, isVirtualRouter); if (result) { return result; } else { @@ -1268,10 +1279,10 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - private boolean attachISOToVM(long vmId, long isoId, boolean attach, boolean forced) { - UserVmVO vm = _userVmDao.findById(vmId); + private boolean attachISOToVM(long vmId, long isoId, boolean attach, boolean forced, boolean isVirtualRouter) { + VirtualMachine vm = !isVirtualRouter ? _userVmDao.findById(vmId) : _vmInstanceDao.findById(vmId); - if (vm == null) { + if (vm == null || (isVirtualRouter && vm.getType() != VirtualMachine.Type.DomainRouter)) { return false; } else if (vm.getState() != State.Running) { return true; @@ -1310,16 +1321,16 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return (a != null && a.getResult()); } - private boolean attachISOToVM(long vmId, long userId, long isoId, boolean attach, boolean forced) { + private boolean attachISOToVM(long vmId, long userId, long isoId, boolean attach, boolean forced, boolean isVirtualRouter) { UserVmVO vm = _userVmDao.findById(vmId); VMTemplateVO iso = _tmpltDao.findById(isoId); - boolean success = attachISOToVM(vmId, isoId, attach, forced); - if (success && attach) { + boolean success = attachISOToVM(vmId, isoId, attach, forced, isVirtualRouter); + if (success && attach && !isVirtualRouter) { vm.setIsoId(iso.getId()); _userVmDao.update(vmId, vm); } - if (success && !attach) { + if (success && !attach && !isVirtualRouter) { vm.setIsoId(null); _userVmDao.update(vmId, vm); } @@ -2129,6 +2140,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Map details = cmd.getDetails(); Account account = CallContext.current().getCallingAccount(); boolean cleanupDetails = cmd.isCleanupDetails(); + Boolean forCks = cmd instanceof UpdateTemplateCmd ? ((UpdateTemplateCmd) cmd).getForCks() : null; CPU.CPUArch arch = cmd.getCPUArch(); // verify that template exists @@ -2178,6 +2190,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, isRoutingTemplate == null && templateType == null && templateTag == null && + forCks == null && arch == null && (! cleanupDetails && details == null) //update details in every case except this one ); @@ -2282,6 +2295,9 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, template.setDetails(details); _tmpltDao.saveDetails(template); } + if (forCks != null) { + template.setForCks(forCks); + } _tmpltDao.update(id, template); diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java b/server/src/main/java/com/cloud/user/AccountManagerImpl.java index 19eba061e13..db79323ed1e 100644 --- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java +++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java @@ -1579,16 +1579,20 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M public void verifyCallerPrivilegeForUserOrAccountOperations(Account userAccount) { logger.debug(String.format("Verifying whether the caller has the correct privileges based on the user's role type and API permissions: %s", userAccount)); - checkCallerRoleTypeAllowedForUserOrAccountOperations(userAccount, null); - checkCallerApiPermissionsForUserOrAccountOperations(userAccount); + if (!Account.Type.PROJECT.equals(userAccount.getType())) { + checkCallerRoleTypeAllowedForUserOrAccountOperations(userAccount, null); + checkCallerApiPermissionsForUserOrAccountOperations(userAccount); + } } protected void verifyCallerPrivilegeForUserOrAccountOperations(User user) { logger.debug(String.format("Verifying whether the caller has the correct privileges based on the user's role type and API permissions: %s", user)); Account userAccount = getAccount(user.getAccountId()); - checkCallerRoleTypeAllowedForUserOrAccountOperations(userAccount, user); - checkCallerApiPermissionsForUserOrAccountOperations(userAccount); + if (!Account.Type.PROJECT.equals(userAccount.getType())) { + checkCallerRoleTypeAllowedForUserOrAccountOperations(userAccount, user); + checkCallerApiPermissionsForUserOrAccountOperations(userAccount); + } } protected void checkCallerRoleTypeAllowedForUserOrAccountOperations(Account userAccount, User user) { @@ -1597,7 +1601,7 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M RoleType userAccountRoleType = getRoleType(userAccount); if (RoleType.Unknown == callerRoleType || RoleType.Unknown == userAccountRoleType) { - String errMsg = String.format("The role type of account [%s, %s] or [%s, %s] is unknown", + String errMsg = String.format("The role type of caller account [%s, %s] or target account [%s, %s] is unknown", callingAccount.getName(), callingAccount.getUuid(), userAccount.getName(), userAccount.getUuid()); throw new PermissionDeniedException(errMsg); } @@ -2707,10 +2711,8 @@ public class AccountManagerImpl extends ManagerBase implements AccountManager, M } } - if (!Account.Type.PROJECT.equals(accountType)) { - AccountVO newAccount = new AccountVO(accountName, domainId, networkDomain, accountType, roleId, uuid); - verifyCallerPrivilegeForUserOrAccountOperations(newAccount); - } + AccountVO newAccount = new AccountVO(accountName, domainId, networkDomain, accountType, roleId, uuid); + verifyCallerPrivilegeForUserOrAccountOperations(newAccount); // Create the account return Transaction.execute(new TransactionCallback<>() { diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 96de70aec81..d08c8655458 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -59,6 +59,7 @@ import javax.naming.ConfigurationException; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.ParserConfigurationException; +import com.cloud.deploy.DeploymentPlan; import com.cloud.network.NetworkService; import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.ControlledEntity.ACLType; @@ -3322,8 +3323,8 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir @Override @ActionEvent(eventType = EventTypes.EVENT_VM_START, eventDescription = "starting Vm", async = true) - public void startVirtualMachine(UserVm vm) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException { - _itMgr.advanceStart(vm.getUuid(), null, null); + public void startVirtualMachine(UserVm vm, DeploymentPlan plan) throws OperationTimedoutException, ResourceUnavailableException, InsufficientCapacityException { + _itMgr.advanceStart(vm.getUuid(), null, plan, null); } @Override diff --git a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java index 3336d44dba8..fc893a7ef50 100644 --- a/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImpl.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeUnit; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.dc.ClusterVO; import org.apache.cloudstack.agent.lb.algorithm.IndirectAgentLBRoundRobinAlgorithm; import org.apache.cloudstack.agent.lb.algorithm.IndirectAgentLBShuffleAlgorithm; import org.apache.cloudstack.agent.lb.algorithm.IndirectAgentLBStaticAlgorithm; @@ -62,7 +63,8 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement public static final ConfigKey IndirectAgentLBAlgorithm = new ConfigKey<>(String.class, "indirect.agent.lb.algorithm", "Advanced", "static", - "The algorithm to be applied on the provided management server list in the 'host' config that that is sent to indirect agents. Allowed values are: static, roundrobin and shuffle.", + "The algorithm to be applied on the provided management server list in the 'host' config that that is sent to indirect agents. Allowed values are: static, roundrobin and shuffle. " + + "Note: The lb algorithm 'shuffle' disables the indirect agent lb check background task once the algorithm is applied on the agent.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, "static,roundrobin,shuffle"); public static final ConfigKey IndirectAgentLBCheckInterval = new ConfigKey<>("Advanced", Long.class, @@ -89,7 +91,9 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement private static final List agentValidResourceStates = List.of( ResourceState.Enabled, ResourceState.Maintenance, ResourceState.Disabled, ResourceState.ErrorInMaintenance, ResourceState.PrepareForMaintenance); - private static final List agentValidHostTypes = List.of(Host.Type.Routing, Host.Type.ConsoleProxy, + private static final List agentNonMaintenanceResourceStates = List.of( + ResourceState.Enabled, ResourceState.Disabled); + public static final List agentValidHostTypes = List.of(Host.Type.Routing, Host.Type.ConsoleProxy, Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM); private static final List agentNonRoutingHostTypes = List.of(Host.Type.ConsoleProxy, Host.Type.SecondaryStorage, Host.Type.SecondaryStorageVM); @@ -132,7 +136,7 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement final org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm algorithm = getAgentMSLBAlgorithm(lbAlgorithm); List hostIdList = orderedHostIdList; if (hostIdList == null) { - hostIdList = algorithm.isHostListNeeded() ? getOrderedHostIdList(dcId) : new ArrayList<>(); + hostIdList = algorithm.isHostListNeeded() ? getOrderedHostIdList(dcId, false) : new ArrayList<>(); } // just in case we have a host in creating state make sure it is in the list: @@ -167,8 +171,8 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement return IndirectAgentLBCheckInterval.valueIn(clusterId); } - List getOrderedHostIdList(final Long dcId) { - final List hostIdList = getAllAgentBasedHostsFromDB(dcId, null); + List getOrderedHostIdList(final Long dcId, boolean excludeHostsInMaintenance) { + final List hostIdList = getAllAgentBasedHostsFromDB(dcId, null, null, excludeHostsInMaintenance); hostIdList.sort(Comparator.comparingLong(x -> x)); return hostIdList; } @@ -259,19 +263,25 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement agentValidResourceStates, agentNonRoutingHostTypes, agentValidHypervisorTypes); } - private List getAllAgentBasedRoutingHostsFromDB(final Long zoneId, final Long clusterId, final Long msId) { + private List getAllAgentBasedRoutingHostsFromDB(final Long zoneId, final Long clusterId, final Long msId, boolean excludeHostsInMaintenance) { + List validResourceStates = excludeHostsInMaintenance ? agentNonMaintenanceResourceStates : agentValidResourceStates; return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, msId, - agentValidResourceStates, List.of(Host.Type.Routing), agentValidHypervisorTypes); + validResourceStates, List.of(Host.Type.Routing), agentValidHypervisorTypes); } - private List getAllAgentBasedHostsFromDB(final Long zoneId, final Long clusterId) { + private List getAllAgentBasedHostsFromDB(final Long zoneId, final Long clusterId, final Long msId, boolean excludeHostsInMaintenance) { + List validResourceStates = excludeHostsInMaintenance ? agentNonMaintenanceResourceStates : agentValidResourceStates; return hostDao.findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(zoneId, clusterId, null, - agentValidResourceStates, agentValidHostTypes, agentValidHypervisorTypes); + validResourceStates, agentValidHostTypes, agentValidHypervisorTypes); + } + + private List getAllAgentBasedHosts(long msId, boolean excludeHostsInMaintenance) { + return getAllAgentBasedHostsFromDB(null, null, msId, excludeHostsInMaintenance); } @Override - public boolean haveAgentBasedHosts(long msId) { - return CollectionUtils.isNotEmpty(getAllAgentBasedHosts(msId)); + public boolean haveAgentBasedHosts(long msId, boolean excludeHostsInMaintenance) { + return CollectionUtils.isNotEmpty(getAllAgentBasedHosts(msId, excludeHostsInMaintenance)); } private org.apache.cloudstack.agent.lb.IndirectAgentLBAlgorithm getAgentMSLBAlgorithm() { @@ -303,8 +313,8 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement //////////////////////////////////////////////////////////// @Override - public void propagateMSListToAgents() { - logger.debug("Propagating management server list update to agents"); + public void propagateMSListToAgents(boolean triggerHostLB) { + logger.debug("Propagating management server list update to the agents"); ExecutorService setupMSListExecutorService = Executors.newFixedThreadPool(10, new NamedThreadFactory("SetupMSList-Worker")); final String lbAlgorithm = getLBAlgorithmName(); final Long globalLbCheckInterval = getLBPreferredHostCheckInterval(null); @@ -316,20 +326,20 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement Map> clusterHostIdsMap = new HashMap<>(); List clusterIds = clusterDao.listAllClusterIds(zone.getId()); for (Long clusterId : clusterIds) { - List hostIds = getAllAgentBasedRoutingHostsFromDB(zone.getId(), clusterId, null); + List hostIds = getAllAgentBasedRoutingHostsFromDB(zone.getId(), clusterId, null, false); clusterHostIdsMap.put(clusterId, hostIds); zoneHostIds.addAll(hostIds); } zoneHostIds.sort(Comparator.comparingLong(x -> x)); final List avoidMsList = mshostDao.listNonUpStateMsIPs(); for (Long nonRoutingHostId : nonRoutingHostIds) { - setupMSListExecutorService.submit(new SetupMSListTask(nonRoutingHostId, zone.getId(), zoneHostIds, avoidMsList, lbAlgorithm, globalLbCheckInterval)); + setupMSListExecutorService.submit(new SetupMSListTask(nonRoutingHostId, zone.getId(), zoneHostIds, avoidMsList, lbAlgorithm, globalLbCheckInterval, triggerHostLB)); } for (Long clusterId : clusterIds) { final Long clusterLbCheckInterval = getLBPreferredHostCheckInterval(clusterId); List hostIds = clusterHostIdsMap.get(clusterId); for (Long hostId : hostIds) { - setupMSListExecutorService.submit(new SetupMSListTask(hostId, zone.getId(), zoneHostIds, avoidMsList, lbAlgorithm, clusterLbCheckInterval)); + setupMSListExecutorService.submit(new SetupMSListTask(hostId, zone.getId(), zoneHostIds, avoidMsList, lbAlgorithm, clusterLbCheckInterval, triggerHostLB)); } } } @@ -345,6 +355,45 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement } } + @Override + public void propagateMSListToAgentsInCluster(Long clusterId) { + if (clusterId == null) { + return; + } + + logger.debug("Propagating management server list update to the agents in cluster " + clusterId); + ClusterVO cluster = clusterDao.findById(clusterId); + if (cluster == null) { + logger.warn("Unable to propagate management server list, couldn't find cluster " + clusterId); + return; + } + DataCenterVO zone = dataCenterDao.findById(cluster.getDataCenterId()); + if (zone == null) { + logger.warn("Unable to propagate management server list, couldn't find zone of the cluster " + clusterId); + return; + } + + ExecutorService setupMSListInClusterExecutorService = Executors.newFixedThreadPool(10, new NamedThreadFactory("SetupMSListInCluster-Worker")); + final String lbAlgorithm = getLBAlgorithmName(); + List clusterHostIds = getAllAgentBasedRoutingHostsFromDB(zone.getId(), clusterId, null, false); + clusterHostIds.sort(Comparator.comparingLong(x -> x)); + final List avoidMsList = mshostDao.listNonUpStateMsIPs(); + final Long clusterLbCheckInterval = getLBPreferredHostCheckInterval(clusterId); + for (Long hostId : clusterHostIds) { + setupMSListInClusterExecutorService.submit(new SetupMSListTask(hostId, zone.getId(), clusterHostIds, avoidMsList, lbAlgorithm, clusterLbCheckInterval, false)); + } + + setupMSListInClusterExecutorService.shutdown(); + try { + if (!setupMSListInClusterExecutorService.awaitTermination(300, TimeUnit.SECONDS)) { + setupMSListInClusterExecutorService.shutdownNow(); + } + } catch (InterruptedException e) { + setupMSListInClusterExecutorService.shutdownNow(); + logger.debug(String.format("Force shutdown setup ms list in cluster service as it did not shutdown in the desired time due to: %s", e.getMessage())); + } + } + private final class SetupMSListTask extends ManagedContextRunnable { private Long hostId; private Long dcId; @@ -352,21 +401,23 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement private List avoidMsList; private String lbAlgorithm; private Long lbCheckInterval; + private Boolean triggerHostLb; public SetupMSListTask(Long hostId, Long dcId, List orderedHostIdList, List avoidMsList, - String lbAlgorithm, Long lbCheckInterval) { + String lbAlgorithm, Long lbCheckInterval, Boolean triggerHostLb) { this.hostId = hostId; this.dcId = dcId; this.orderedHostIdList = orderedHostIdList; this.avoidMsList = avoidMsList; this.lbAlgorithm = lbAlgorithm; this.lbCheckInterval = lbCheckInterval; + this.triggerHostLb = triggerHostLb; } @Override protected void runInContext() { final List msList = getManagementServerList(hostId, dcId, orderedHostIdList); - final SetupMSListCommand cmd = new SetupMSListCommand(msList, avoidMsList, lbAlgorithm, lbCheckInterval); + final SetupMSListCommand cmd = new SetupMSListCommand(msList, avoidMsList, lbAlgorithm, lbCheckInterval, triggerHostLb); cmd.setWait(60); final Answer answer = agentManager.easySend(hostId, cmd); if (answer == null || !answer.getResult()) { @@ -419,9 +470,9 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement protected boolean migrateRoutingHostAgentsInCluster(long clusterId, String fromMsUuid, long fromMsId, DataCenter dc, long migrationStartTimeInMs, long timeoutDurationInMs, final List avoidMsList, String lbAlgorithm, - boolean lbAlgorithmChanged, List orderedHostIdList) { + boolean lbAlgorithmChanged, List orderedHostIdList, boolean excludeHostsInMaintenance) { - List agentBasedHostsOfMsInDcAndCluster = getAllAgentBasedRoutingHostsFromDB(dc.getId(), clusterId, fromMsId); + List agentBasedHostsOfMsInDcAndCluster = getAllAgentBasedRoutingHostsFromDB(dc.getId(), clusterId, fromMsId, excludeHostsInMaintenance); if (CollectionUtils.isEmpty(agentBasedHostsOfMsInDcAndCluster)) { return true; } @@ -461,7 +512,7 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement } @Override - public boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorithm, long timeoutDurationInMs) { + public boolean migrateAgents(String fromMsUuid, long fromMsId, String lbAlgorithm, long timeoutDurationInMs, boolean excludeHostsInMaintenance) { if (timeoutDurationInMs <= 0) { logger.debug(String.format("Not migrating indirect agents from management server node %d (id: %s) to other nodes, invalid timeout duration", fromMsId, fromMsUuid)); return false; @@ -469,7 +520,7 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement logger.debug(String.format("Migrating indirect agents from management server node %d (id: %s) to other nodes", fromMsId, fromMsUuid)); long migrationStartTimeInMs = System.currentTimeMillis(); - if (!haveAgentBasedHosts(fromMsId)) { + if (!haveAgentBasedHosts(fromMsId, excludeHostsInMaintenance)) { logger.info(String.format("No indirect agents available on management server node %d (id: %s), to migrate", fromMsId, fromMsUuid)); return true; } @@ -489,7 +540,7 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement List dataCenterList = dcDao.listAll(); for (DataCenterVO dc : dataCenterList) { if (!migrateAgentsInZone(dc, fromMsUuid, fromMsId, avoidMsList, lbAlgorithm, lbAlgorithmChanged, - migrationStartTimeInMs, timeoutDurationInMs)) { + migrationStartTimeInMs, timeoutDurationInMs, excludeHostsInMaintenance)) { return false; } } @@ -498,8 +549,8 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement } private boolean migrateAgentsInZone(DataCenterVO dc, String fromMsUuid, long fromMsId, List avoidMsList, - String lbAlgorithm, boolean lbAlgorithmChanged, long migrationStartTimeInMs, long timeoutDurationInMs) { - List orderedHostIdList = getOrderedHostIdList(dc.getId()); + String lbAlgorithm, boolean lbAlgorithmChanged, long migrationStartTimeInMs, long timeoutDurationInMs, boolean excludeHostsInMaintenance) { + List orderedHostIdList = getOrderedHostIdList(dc.getId(), excludeHostsInMaintenance); if (!migrateNonRoutingHostAgentsInZone(fromMsUuid, fromMsId, dc, migrationStartTimeInMs, timeoutDurationInMs, avoidMsList, lbAlgorithm, lbAlgorithmChanged, orderedHostIdList)) { return false; @@ -507,7 +558,7 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement List clusterIds = clusterDao.listAllClusterIds(dc.getId()); for (Long clusterId : clusterIds) { if (!migrateRoutingHostAgentsInCluster(clusterId, fromMsUuid, fromMsId, dc, migrationStartTimeInMs, - timeoutDurationInMs, avoidMsList, lbAlgorithm, lbAlgorithmChanged, orderedHostIdList)) { + timeoutDurationInMs, avoidMsList, lbAlgorithm, lbAlgorithmChanged, orderedHostIdList, excludeHostsInMaintenance)) { return false; } } @@ -547,7 +598,9 @@ public class IndirectAgentLBServiceImpl extends ComponentLifecycleBase implement final MigrateAgentConnectionCommand cmd = new MigrateAgentConnectionCommand(msList, avoidMsList, lbAlgorithm, lbCheckInterval); cmd.setWait(60); final Answer answer = agentManager.easySend(hostId, cmd); //may not receive answer when the agent disconnects immediately and try reconnecting to other ms host - if (answer != null && !answer.getResult()) { + if (answer == null) { + logger.warn(String.format("Got empty answer while initiating migration of agent connection for host agent ID: %d", hostId)); + } else if (!answer.getResult()) { logger.warn(String.format("Error while initiating migration of agent connection for host agent ID: %d - %s", hostId, answer.getDetails())); } updateLastManagementServer(hostId, fromMsId); diff --git a/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java b/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java index a68623aa144..a5fc791de99 100644 --- a/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java +++ b/server/src/test/java/com/cloud/api/ApiResponseHelperTest.java @@ -16,37 +16,36 @@ // under the License. package com.cloud.api; -import com.cloud.capacity.Capacity; -import com.cloud.configuration.Resource; -import com.cloud.domain.DomainVO; -import com.cloud.network.PublicIpQuarantine; -import com.cloud.network.as.AutoScaleVmGroup; -import com.cloud.network.as.AutoScaleVmGroupVO; -import com.cloud.network.as.AutoScaleVmProfileVO; -import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; -import com.cloud.network.dao.IPAddressDao; -import com.cloud.network.dao.IPAddressVO; -import com.cloud.network.dao.LoadBalancerVO; -import com.cloud.network.dao.NetworkServiceMapDao; -import com.cloud.network.dao.NetworkVO; -import com.cloud.storage.VMTemplateVO; -import com.cloud.usage.UsageVO; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.User; -import com.cloud.user.UserData; -import com.cloud.user.UserDataVO; -import com.cloud.user.UserVO; -import com.cloud.user.dao.UserDataDao; -import com.cloud.utils.net.Ip; -import com.cloud.vm.NicSecondaryIp; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; + import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.response.AutoScaleVmGroupResponse; import org.apache.cloudstack.api.response.AutoScaleVmProfileResponse; import org.apache.cloudstack.api.response.DirectDownloadCertificateResponse; +import org.apache.cloudstack.api.response.GuestOSCategoryResponse; import org.apache.cloudstack.api.response.IpQuarantineResponse; import org.apache.cloudstack.api.response.NicSecondaryIpResponse; +import org.apache.cloudstack.api.response.ResourceIconResponse; +import org.apache.cloudstack.api.response.TemplateResponse; import org.apache.cloudstack.api.response.UnmanagedInstanceResponse; import org.apache.cloudstack.api.response.UsageRecordResponse; import org.apache.cloudstack.context.CallContext; @@ -63,21 +62,38 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; -import java.lang.reflect.Field; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.TimeZone; -import java.util.UUID; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.when; +import com.cloud.capacity.Capacity; +import com.cloud.configuration.Resource; +import com.cloud.domain.DomainVO; +import com.cloud.network.PublicIpQuarantine; +import com.cloud.network.as.AutoScaleVmGroup; +import com.cloud.network.as.AutoScaleVmGroupVO; +import com.cloud.network.as.AutoScaleVmProfileVO; +import com.cloud.network.as.dao.AutoScaleVmGroupVmMapDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.LoadBalancerVO; +import com.cloud.network.dao.NetworkServiceMapDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.resource.icon.ResourceIconVO; +import com.cloud.server.ResourceIcon; +import com.cloud.server.ResourceIconManager; +import com.cloud.server.ResourceTag; +import com.cloud.storage.GuestOsCategory; +import com.cloud.storage.VMTemplateVO; +import com.cloud.usage.UsageVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.User; +import com.cloud.user.UserData; +import com.cloud.user.UserDataVO; +import com.cloud.user.UserVO; +import com.cloud.user.dao.UserDataDao; +import com.cloud.utils.net.Ip; +import com.cloud.vm.NicSecondaryIp; @RunWith(MockitoJUnitRunner.class) public class ApiResponseHelperTest { @@ -105,6 +121,9 @@ public class ApiResponseHelperTest { @Mock IPAddressDao ipAddressDaoMock; + @Mock + ResourceIconManager resourceIconManager; + @Spy @InjectMocks ApiResponseHelper apiResponseHelper = new ApiResponseHelper(); @@ -481,4 +500,135 @@ public class ApiResponseHelperTest { Assert.assertTrue(apiResponseHelper.capacityListingForSingleNonGpuType(List.of(c1, c2))); Assert.assertFalse(apiResponseHelper.capacityListingForSingleNonGpuType(List.of(c1, c2, c3))); } + + @Test + public void testCreateGuestOSCategoryResponse_WithResourceIcon() { + GuestOsCategory guestOsCategory = Mockito.mock(GuestOsCategory.class); + ResourceIconVO resourceIconVO = Mockito.mock(ResourceIconVO.class); + String uuid = UUID.randomUUID().toString(); + String name = "Ubuntu"; + boolean featured = true; + Mockito.when(guestOsCategory.getUuid()).thenReturn(uuid); + Mockito.when(guestOsCategory.getName()).thenReturn(name); + Mockito.when(guestOsCategory.isFeatured()).thenReturn(featured); + ResourceIconResponse mockIconResponse = Mockito.mock(ResourceIconResponse.class); + try (MockedStatic ignored = Mockito.mockStatic(ApiDBUtils.class)) { + Mockito.when(ApiDBUtils.getResourceIconByResourceUUID(uuid, ResourceTag.ResourceObjectType.GuestOsCategory)).thenReturn(resourceIconVO); + Mockito.when(ApiDBUtils.newResourceIconResponse(resourceIconVO)).thenReturn(mockIconResponse); + GuestOSCategoryResponse response = apiResponseHelper.createGuestOSCategoryResponse(guestOsCategory); + Assert.assertNotNull(response); + Assert.assertEquals(uuid, response.getId()); + Assert.assertEquals(name, response.getName()); + Object obj = ReflectionTestUtils.getField(response, "featured"); + if (obj == null) { + Assert.fail("Invalid featured value"); + } + Assert.assertTrue((Boolean)obj); + obj = ReflectionTestUtils.getField(response, "resourceIconResponse"); + Assert.assertNotNull(obj); + Assert.assertEquals("oscategory", response.getObjectName()); + } + } + + @Test + public void testCreateGuestOSCategoryResponse_WithoutResourceIcon() { + GuestOsCategory guestOsCategory = Mockito.mock(GuestOsCategory.class); + String uuid = "1234"; + String name = "Ubuntu"; + boolean featured = false; + Mockito.when(guestOsCategory.getUuid()).thenReturn(uuid); + Mockito.when(guestOsCategory.getName()).thenReturn(name); + Mockito.when(guestOsCategory.isFeatured()).thenReturn(featured); + try (MockedStatic ignored = Mockito.mockStatic(ApiDBUtils.class)) { + when(ApiDBUtils.getResourceIconByResourceUUID(uuid, ResourceTag.ResourceObjectType.GuestOsCategory)).thenReturn(null); + GuestOSCategoryResponse response = apiResponseHelper.createGuestOSCategoryResponse(guestOsCategory); + Assert.assertNotNull(response); + Assert.assertEquals(uuid, response.getId()); + Assert.assertEquals(name, response.getName()); + Object obj = ReflectionTestUtils.getField(response, "featured"); + if (obj == null) { + Assert.fail("Invalid featured value"); + } + Assert.assertFalse((Boolean)obj); + obj = ReflectionTestUtils.getField(response, "resourceIconResponse"); + Assert.assertNull(obj); + Assert.assertEquals("oscategory", response.getObjectName()); + } + } + + @Test + public void testCreateGuestOSCategoryResponse_WithShowIconFalse() { + GuestOsCategory guestOsCategory = Mockito.mock(GuestOsCategory.class); + Mockito.when(guestOsCategory.getUuid()).thenReturn(UUID.randomUUID().toString()); + try (MockedStatic mockedStatic = Mockito.mockStatic(ApiDBUtils.class)) { + GuestOSCategoryResponse response = apiResponseHelper.createGuestOSCategoryResponse(guestOsCategory, false); + Assert.assertNotNull(response); + mockedStatic.verify(() -> ApiDBUtils.getResourceIconByResourceUUID(Mockito.any(), Mockito.any()), + Mockito.never()); + } + } + + @Test + public void testGetResourceIconsUsingOsCategory_withValidData() { + TemplateResponse template1 = Mockito.mock(TemplateResponse.class); + when(template1.getId()).thenReturn("t1"); + when(template1.getOsTypeCategoryId()).thenReturn(100L); + TemplateResponse template2 = Mockito.mock(TemplateResponse.class); + when(template2.getId()).thenReturn("t2"); + when(template2.getOsTypeCategoryId()).thenReturn(200L); + List responses = Arrays.asList(template1, template2); + Map icons = new HashMap<>(); + ResourceIcon icon1 = Mockito.mock(ResourceIcon.class); + ResourceIcon icon2 = Mockito.mock(ResourceIcon.class); + icons.put(100L, icon1); + icons.put(200L, icon2); + when(resourceIconManager.getByResourceTypeAndIds(Mockito.eq(ResourceTag.ResourceObjectType.GuestOsCategory), Mockito.anySet())) + .thenReturn(icons); + Map result = apiResponseHelper.getResourceIconsUsingOsCategory(responses); + assertEquals(2, result.size()); + assertEquals(icon1, result.get("t1")); + assertEquals(icon2, result.get("t2")); + } + + @Test + public void testGetResourceIconsUsingOsCategory_missingIcons() { + TemplateResponse template1 = Mockito.mock(TemplateResponse.class); + when(template1.getId()).thenReturn("t1"); + when(template1.getOsTypeCategoryId()).thenReturn(100L); + List responses = List.of(template1); + when(resourceIconManager.getByResourceTypeAndIds(Mockito.eq(ResourceTag.ResourceObjectType.GuestOsCategory), Mockito.anySet())).thenReturn(Collections.emptyMap()); + Map result = apiResponseHelper.getResourceIconsUsingOsCategory(responses); + assertTrue(result.containsKey("t1")); + assertNull(result.get("t1")); + } + + @Test + public void testUpdateTemplateIsoResponsesForIcons_withMixedIcons() { + TemplateResponse template1 = Mockito.mock(TemplateResponse.class); + when(template1.getId()).thenReturn("t1"); + TemplateResponse template2 = Mockito.mock(TemplateResponse.class); + when(template2.getId()).thenReturn("t2"); + List responses = Arrays.asList(template1, template2); + Map isoIcons = new HashMap<>(); + isoIcons.put("t1", Mockito.mock(ResourceIcon.class)); + when(resourceIconManager.getByResourceTypeAndUuids(ResourceTag.ResourceObjectType.ISO, Set.of("t1", "t2"))) + .thenReturn(isoIcons); + Map fallbackIcons = Map.of("t2", Mockito.mock(ResourceIcon.class)); + Mockito.doReturn(fallbackIcons).when(apiResponseHelper).getResourceIconsUsingOsCategory(Mockito.anyList()); + ResourceIconResponse iconResponse1 = new ResourceIconResponse(); + ResourceIconResponse iconResponse2 = new ResourceIconResponse(); + Mockito.doReturn(iconResponse1).when(apiResponseHelper).createResourceIconResponse(isoIcons.get("t1")); + Mockito.doReturn(iconResponse2).when(apiResponseHelper).createResourceIconResponse(fallbackIcons.get("t2")); + apiResponseHelper.updateTemplateIsoResponsesForIcons(responses, ResourceTag.ResourceObjectType.ISO); + verify(template1).setResourceIconResponse(iconResponse1); + verify(template2).setResourceIconResponse(iconResponse2); + } + + @Test + public void testUpdateTemplateIsoResponsesForIcons_emptyInput() { + apiResponseHelper.updateTemplateIsoResponsesForIcons(Collections.emptyList(), + ResourceTag.ResourceObjectType.Template); + Mockito.verify(resourceIconManager, Mockito.never()).getByResourceTypeAndUuids(Mockito.any(), + Mockito.anyCollection()); + } } diff --git a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java index 5b0f10f8508..e83363877aa 100644 --- a/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java +++ b/server/src/test/java/com/cloud/configuration/ConfigurationManagerImplTest.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.configuration; +import com.cloud.alert.AlertManager; import com.cloud.capacity.dao.CapacityDao; import com.cloud.dc.DataCenterVO; import com.cloud.dc.VlanVO; @@ -53,6 +54,7 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.annotation.dao.AnnotationDao; import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd; import org.apache.cloudstack.api.command.admin.network.CreateNetworkOfferingCmd; @@ -930,4 +932,113 @@ public class ConfigurationManagerImplTest { configurationManagerImplSpy.validateConfigurationAllowedOnlyForDefaultAdmin(AccountManagerImpl.listOfRoleTypesAllowedForOperationsOfSameRoleType.key(), invalidValue); } } + + + @Test + public void getConfigurationTypeWrapperClassTestReturnsConfigType() { + Config configuration = Config.AlertEmailAddresses; + + Assert.assertEquals(configuration.getType(), configurationManagerImplSpy.getConfigurationTypeWrapperClass(configuration.key())); + } + + @Test + public void getConfigurationTypeWrapperClassTestReturnsConfigKeyType() { + String configurationName = "configuration.name"; + + Mockito.when(configDepot.get(configurationName)).thenReturn(configKeyMock); + Mockito.when(configKeyMock.type()).thenReturn(Integer.class); + + Assert.assertEquals(Integer.class, configurationManagerImplSpy.getConfigurationTypeWrapperClass(configurationName)); + } + + @Test + public void getConfigurationTypeWrapperClassTestReturnsNullWhenConfigurationDoesNotExist() { + String configurationName = "configuration.name"; + + Mockito.when(configDepot.get(configurationName)).thenReturn(null); + Assert.assertNull(configurationManagerImplSpy.getConfigurationTypeWrapperClass(configurationName)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsStringWhenTypeIsNull() { + Assert.assertEquals(Configuration.ValueType.String.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(null, null)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsStringWhenTypeIsStringAndConfigurationKindIsNull() { + Mockito.when(configurationVOMock.getKind()).thenReturn(null); + Assert.assertEquals(Configuration.ValueType.String.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(String.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsKindWhenTypeIsStringAndKindIsNotNull() { + Mockito.when(configurationVOMock.getKind()).thenReturn(ConfigKey.Kind.CSV.name()); + Assert.assertEquals(ConfigKey.Kind.CSV.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(String.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsKindWhenTypeIsCharacterAndKindIsNotNull() { + Mockito.when(configurationVOMock.getKind()).thenReturn(ConfigKey.Kind.CSV.name()); + Assert.assertEquals(ConfigKey.Kind.CSV.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Character.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsNumberWhenTypeIsInteger() { + Assert.assertEquals(Configuration.ValueType.Number.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Integer.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsNumberWhenTypeIsLong() { + Assert.assertEquals(Configuration.ValueType.Number.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Long.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsNumberWhenTypeIsShort() { + Assert.assertEquals(Configuration.ValueType.Number.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Short.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsDecimalWhenTypeIsFloat() { + Assert.assertEquals(Configuration.ValueType.Decimal.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Float.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsDecimalWhenTypeIsDouble() { + Assert.assertEquals(Configuration.ValueType.Decimal.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Double.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsBooleanWhenTypeIsBoolean() { + Assert.assertEquals(Configuration.ValueType.Boolean.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Boolean.class, configurationVOMock)); + } + + @Test + public void parseConfigurationTypeIntoStringTestReturnsStringWhenTypeDoesNotMatchAnyAvailableType() { + Assert.assertEquals(Configuration.ValueType.String.name(), configurationManagerImplSpy.parseConfigurationTypeIntoString(Object.class, configurationVOMock)); + } + + @Test + public void getConfigurationTypeTestReturnsStringWhenConfigurationDoesNotExist() { + Mockito.when(configDao.findByName(Mockito.anyString())).thenReturn(null); + Assert.assertEquals(Configuration.ValueType.String.name(), configurationManagerImplSpy.getConfigurationType(Mockito.anyString())); + } + + @Test + public void getConfigurationTypeTestReturnsRangeForConfigurationsThatAcceptIntervals() { + String configurationName = AlertManager.CPUCapacityThreshold.key(); + + Mockito.when(configDao.findByName(configurationName)).thenReturn(configurationVOMock); + Assert.assertEquals(Configuration.ValueType.Range.name(), configurationManagerImplSpy.getConfigurationType(configurationName)); + } + + @Test + public void getConfigurationTypeTestReturnsStringRepresentingConfigurationType() { + ConfigKey configuration = RoleService.EnableDynamicApiChecker; + + Mockito.when(configDao.findByName(configuration.key())).thenReturn(configurationVOMock); + Mockito.doReturn(configuration.type()).when(configurationManagerImplSpy).getConfigurationTypeWrapperClass(configuration.key()); + + configurationManagerImplSpy.getConfigurationType(configuration.key()); + Mockito.verify(configurationManagerImplSpy).parseConfigurationTypeIntoString(configuration.type(), configurationVOMock); + } } diff --git a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java index ee56a092dd1..8d513619805 100644 --- a/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java +++ b/server/src/test/java/com/cloud/network/vpc/VpcManagerImplTest.java @@ -492,7 +492,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], null, null, null, true, 1500, null, null, null); + ip4Dns[0], null, null, null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -504,7 +504,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], ip4Dns[1], ip6Dns[0], null, true, 1500, null, null, null); + ip4Dns[0], ip4Dns[1], ip6Dns[0], null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -519,7 +519,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null); + ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -536,7 +536,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, ip4Cidr, vpcDomain, - ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null); + ip4Dns[0], ip4Dns[1], null, null, true, 1500, null, null, null, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } @@ -559,7 +559,7 @@ public class VpcManagerImplTest { try { doNothing().when(resourceLimitService).checkResourceLimit(account, Resource.ResourceType.vpc); manager.createVpc(zoneId, vpcOfferingId, vpcOwnerId, vpcName, vpcName, null, vpcDomain, - ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds); + ip4Dns[0], ip4Dns[1], null, null, true, 1500, 24, null, bgpPeerIds, false); } catch (ResourceAllocationException e) { Assert.fail(String.format("failure with exception: %s", e.getMessage())); } diff --git a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java index 8b9928733ec..bdce0a5ae9d 100644 --- a/server/src/test/java/com/cloud/server/ManagementServerImplTest.java +++ b/server/src/test/java/com/cloud/server/ManagementServerImplTest.java @@ -16,6 +16,56 @@ // under the License. package com.cloud.server; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.apache.cloudstack.annotation.dao.AnnotationDao; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; +import org.apache.cloudstack.api.command.admin.guest.AddGuestOsCategoryCmd; +import org.apache.cloudstack.api.command.admin.guest.DeleteGuestOsCategoryCmd; +import org.apache.cloudstack.api.command.admin.guest.UpdateGuestOsCategoryCmd; +import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; +import org.apache.cloudstack.api.command.user.guest.ListGuestOsCategoriesCmd; +import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd; +import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; +import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; +import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; +import org.apache.cloudstack.config.Configuration; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; +import org.apache.cloudstack.framework.config.ConfigDepot; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.cloudstack.framework.config.impl.ConfigurationVO; +import org.apache.cloudstack.userdata.UserDataManager; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.cpu.CPU; import com.cloud.dc.Vlan.VlanType; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; @@ -26,7 +76,12 @@ import com.cloud.host.dao.HostDetailsDao; import com.cloud.network.IpAddress; import com.cloud.network.IpAddressManagerImpl; import com.cloud.network.dao.IPAddressVO; +import com.cloud.storage.GuestOSCategoryVO; +import com.cloud.storage.GuestOSVO; +import com.cloud.storage.GuestOsCategory; import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.GuestOSCategoryDao; +import com.cloud.storage.dao.GuestOSDao; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; import com.cloud.user.AccountManager; @@ -46,46 +101,6 @@ import com.cloud.vm.UserVmDetailVO; import com.cloud.vm.UserVmVO; import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.UserVmDetailsDao; -import org.apache.cloudstack.annotation.dao.AnnotationDao; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.BaseCmd; -import org.apache.cloudstack.api.command.admin.config.ListCfgsByCmd; -import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd; -import org.apache.cloudstack.api.command.user.ssh.RegisterSSHKeyPairCmd; -import org.apache.cloudstack.api.command.user.userdata.DeleteUserDataCmd; -import org.apache.cloudstack.api.command.user.userdata.ListUserDataCmd; -import org.apache.cloudstack.api.command.user.userdata.RegisterUserDataCmd; -import org.apache.cloudstack.config.Configuration; -import org.apache.cloudstack.context.CallContext; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStore; -import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; -import org.apache.cloudstack.framework.config.ConfigDepot; -import org.apache.cloudstack.framework.config.ConfigKey; -import org.apache.cloudstack.framework.config.dao.ConfigurationDao; -import org.apache.cloudstack.framework.config.impl.ConfigurationVO; -import org.apache.cloudstack.userdata.UserDataManager; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.test.util.ReflectionTestUtils; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.nullable; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.lenient; -import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) public class ManagementServerImplTest { @@ -118,7 +133,7 @@ public class ManagementServerImplTest { UserDataDao _userDataDao; @Mock - VMTemplateDao _templateDao; + VMTemplateDao templateDao; @Mock AnnotationDao annotationDao; @@ -129,9 +144,6 @@ public class ManagementServerImplTest { @Mock UserDataManager userDataManager; - @Spy - ManagementServerImpl spy = new ManagementServerImpl(); - @Mock UserVmDetailsDao userVmDetailsDao; @@ -147,23 +159,22 @@ public class ManagementServerImplTest { @Mock DomainDao domainDao; + @Mock + GuestOSCategoryDao _guestOSCategoryDao; + + @Mock + GuestOSDao _guestOSDao; + + @Spy + @InjectMocks + ManagementServerImpl spy = new ManagementServerImpl(); + private AutoCloseable closeable; @Before public void setup() throws IllegalAccessException, NoSuchFieldException { closeable = MockitoAnnotations.openMocks(this); CallContext.register(Mockito.mock(User.class), Mockito.mock(Account.class)); - spy._accountMgr = _accountMgr; - spy.userDataDao = _userDataDao; - spy.templateDao = _templateDao; - spy._userVmDao = _userVmDao; - spy.annotationDao = annotationDao; - spy._UserVmDetailsDao = userVmDetailsDao; - spy._detailsDao = hostDetailsDao; - spy.userDataManager = userDataManager; - spy._configDao = configDao; - spy._configDepot = configDepot; - spy._domainDao = domainDao; } @After @@ -407,7 +418,7 @@ public class ManagementServerImplTest { Mockito.when(userData.getId()).thenReturn(1L); when(_userDataDao.findById(1L)).thenReturn(userData); - when(_templateDao.findTemplatesLinkedToUserdata(1L)).thenReturn(new ArrayList()); + when(templateDao.findTemplatesLinkedToUserdata(1L)).thenReturn(new ArrayList()); when(_userVmDao.findByUserDataId(1L)).thenReturn(new ArrayList()); when(_userDataDao.remove(1L)).thenReturn(true); @@ -437,7 +448,7 @@ public class ManagementServerImplTest { VMTemplateVO vmTemplateVO = Mockito.mock(VMTemplateVO.class); List linkedTemplates = new ArrayList<>(); linkedTemplates.add(vmTemplateVO); - when(_templateDao.findTemplatesLinkedToUserdata(1L)).thenReturn(linkedTemplates); + when(templateDao.findTemplatesLinkedToUserdata(1L)).thenReturn(linkedTemplates); spy.deleteUserData(cmd); } @@ -461,7 +472,7 @@ public class ManagementServerImplTest { Mockito.when(userData.getId()).thenReturn(1L); when(_userDataDao.findById(1L)).thenReturn(userData); - when(_templateDao.findTemplatesLinkedToUserdata(1L)).thenReturn(new ArrayList()); + when(templateDao.findTemplatesLinkedToUserdata(1L)).thenReturn(new ArrayList()); UserVmVO userVmVO = Mockito.mock(UserVmVO.class); List vms = new ArrayList<>(); @@ -498,7 +509,7 @@ public class ManagementServerImplTest { Pair, Integer> result = new Pair(userDataList, 1); when(_userDataDao.searchAndCount(nullable(SearchCriteria.class), nullable(Filter.class))).thenReturn(result); - Pair, Integer> userdataResultList = spy.listUserDatas(cmd); + Pair, Integer> userdataResultList = spy.listUserDatas(cmd, false); Assert.assertEquals(userdataResultList.first().get(0), userDataList.get(0)); } @@ -531,7 +542,7 @@ public class ManagementServerImplTest { Pair, Integer> result = new Pair(userDataList, 1); when(_userDataDao.searchAndCount(nullable(SearchCriteria.class), nullable(Filter.class))).thenReturn(result); - Pair, Integer> userdataResultList = spy.listUserDatas(cmd); + Pair, Integer> userdataResultList = spy.listUserDatas(cmd, false); Assert.assertEquals(userdataResultList.first().get(0), userDataList.get(0)); } @@ -564,7 +575,7 @@ public class ManagementServerImplTest { Pair, Integer> result = new Pair(userDataList, 1); when(_userDataDao.searchAndCount(nullable(SearchCriteria.class), nullable(Filter.class))).thenReturn(result); - Pair, Integer> userdataResultList = spy.listUserDatas(cmd); + Pair, Integer> userdataResultList = spy.listUserDatas(cmd, false); Assert.assertEquals(userdataResultList.first().get(0), userDataList.get(0)); } @@ -740,4 +751,272 @@ public class ManagementServerImplTest { Assert.assertEquals("0.85", result.first().get(0).getValue()); } + @Test + public void testAddGuestOsCategory() { + AddGuestOsCategoryCmd addCmd = Mockito.mock(AddGuestOsCategoryCmd.class); + String name = "Ubuntu"; + boolean featured = true; + Mockito.when(addCmd.getName()).thenReturn(name); + Mockito.when(addCmd.isFeatured()).thenReturn(featured); + Mockito.doAnswer((Answer) invocation -> (GuestOSCategoryVO)invocation.getArguments()[0]).when(_guestOSCategoryDao).persist(Mockito.any(GuestOSCategoryVO.class)); + GuestOsCategory result = spy.addGuestOsCategory(addCmd); + Assert.assertNotNull(result); + Assert.assertEquals(name, result.getName()); + Assert.assertEquals(featured, result.isFeatured()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).persist(any(GuestOSCategoryVO.class)); + } + + @Test + public void testUpdateGuestOsCategory() { + UpdateGuestOsCategoryCmd updateCmd = Mockito.mock(UpdateGuestOsCategoryCmd.class); + GuestOSCategoryVO guestOSCategory = new GuestOSCategoryVO("Old name", false); + long id = 1L; + String name = "Updated Name"; + Boolean featured = true; + Integer sortKey = 10; + Mockito.when(updateCmd.getId()).thenReturn(id); + Mockito.when(updateCmd.getName()).thenReturn(name); + Mockito.when(updateCmd.isFeatured()).thenReturn(featured); + Mockito.when(updateCmd.getSortKey()).thenReturn(sortKey); + Mockito.when(_guestOSCategoryDao.findById(id)).thenReturn(guestOSCategory); + Mockito.when(_guestOSCategoryDao.update(Mockito.eq(id), any(GuestOSCategoryVO.class))).thenReturn(true); + GuestOsCategory result = spy.updateGuestOsCategory(updateCmd); + Assert.assertNotNull(result); + Assert.assertEquals(name, result.getName()); + Assert.assertEquals(featured, result.isFeatured()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).findById(id); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).update(Mockito.eq(id), any(GuestOSCategoryVO.class)); + } + + @Test(expected = InvalidParameterValueException.class) + public void testUpdateGuestOsCategory_ThrowsExceptionWhenCategoryNotFound() { + UpdateGuestOsCategoryCmd updateCmd = Mockito.mock(UpdateGuestOsCategoryCmd.class); + long id = 1L; + when(updateCmd.getId()).thenReturn(id); + when(_guestOSCategoryDao.findById(id)).thenReturn(null); + spy.updateGuestOsCategory(updateCmd); + } + + @Test + public void testUpdateGuestOsCategory_NoChanges() { + UpdateGuestOsCategoryCmd updateCmd = Mockito.mock(UpdateGuestOsCategoryCmd.class); + GuestOSCategoryVO guestOSCategory = new GuestOSCategoryVO("Old name", false); + long id = 1L; + when(updateCmd.getId()).thenReturn(id); + when(updateCmd.getName()).thenReturn(null); + when(updateCmd.isFeatured()).thenReturn(null); + when(updateCmd.getSortKey()).thenReturn(null); + when(_guestOSCategoryDao.findById(id)).thenReturn(guestOSCategory); + GuestOsCategory result = spy.updateGuestOsCategory(updateCmd); + Assert.assertNotNull(result); + Assert.assertNotNull(result.getName()); + Assert.assertFalse(result.isFeatured()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).findById(id); + Mockito.verify(_guestOSCategoryDao, Mockito.never()).update(Mockito.eq(id), any(GuestOSCategoryVO.class)); + } + + @Test + public void testUpdateGuestOsCategory_UpdateNameOnly() { + UpdateGuestOsCategoryCmd updateCmd = Mockito.mock(UpdateGuestOsCategoryCmd.class); + GuestOSCategoryVO guestOSCategory = new GuestOSCategoryVO("Old name", false); + long id = 1L; + String name = "Updated Name"; + Mockito.when(updateCmd.getId()).thenReturn(id); + Mockito.when(updateCmd.getName()).thenReturn(name); + Mockito.when(updateCmd.isFeatured()).thenReturn(null); + Mockito.when(updateCmd.getSortKey()).thenReturn(null); + Mockito.when(_guestOSCategoryDao.findById(id)).thenReturn(guestOSCategory); + Mockito.when(_guestOSCategoryDao.update(Mockito.eq(id), any(GuestOSCategoryVO.class))).thenReturn(true); + GuestOsCategory result = spy.updateGuestOsCategory(updateCmd); + Assert.assertNotNull(result); + Assert.assertEquals(name, result.getName()); + Assert.assertFalse(result.isFeatured()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).findById(id); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).update(Mockito.eq(id), any(GuestOSCategoryVO.class)); + } + + @Test + public void testDeleteGuestOsCategory_Successful() { + DeleteGuestOsCategoryCmd deleteCmd = Mockito.mock(DeleteGuestOsCategoryCmd.class); + GuestOSCategoryVO guestOSCategory = Mockito.mock(GuestOSCategoryVO.class); + long id = 1L; + Mockito.when(deleteCmd.getId()).thenReturn(id); + Mockito.when(_guestOSCategoryDao.findById(id)).thenReturn(guestOSCategory); + Mockito.when(_guestOSDao.listIdsByCategoryId(id)).thenReturn(Arrays.asList()); + Mockito.when(_guestOSCategoryDao.remove(id)).thenReturn(true); + boolean result = spy.deleteGuestOsCategory(deleteCmd); + Assert.assertTrue(result); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).findById(id); + Mockito.verify(_guestOSDao, Mockito.times(1)).listIdsByCategoryId(id); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).remove(id); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteGuestOsCategory_ThrowsExceptionWhenCategoryNotFound() { + DeleteGuestOsCategoryCmd deleteCmd = Mockito.mock(DeleteGuestOsCategoryCmd.class); + long id = 1L; + Mockito.when(deleteCmd.getId()).thenReturn(id); + Mockito.when(_guestOSCategoryDao.findById(id)).thenReturn(null); + spy.deleteGuestOsCategory(deleteCmd); + } + + @Test(expected = InvalidParameterValueException.class) + public void testDeleteGuestOsCategory_ThrowsExceptionWhenGuestOsExists() { + DeleteGuestOsCategoryCmd deleteCmd = Mockito.mock(DeleteGuestOsCategoryCmd.class); + GuestOSCategoryVO guestOSCategory = Mockito.mock(GuestOSCategoryVO.class); + long id = 1L; + Mockito.when(deleteCmd.getId()).thenReturn(id); + Mockito.when(_guestOSCategoryDao.findById(id)).thenReturn(guestOSCategory); + Mockito.when(_guestOSDao.listIdsByCategoryId(id)).thenReturn(Arrays.asList(1L)); + spy.deleteGuestOsCategory(deleteCmd); + } + + private void mockGuestOsJoin() { + GuestOSVO vo = mock(GuestOSVO.class); + SearchBuilder sb = mock(SearchBuilder.class); + when(sb.entity()).thenReturn(vo); + when(_guestOSDao.createSearchBuilder()).thenReturn(sb); + } + + @Test + public void testListGuestOSCategoriesByCriteria_Success() { + ListGuestOsCategoriesCmd listCmd = Mockito.mock(ListGuestOsCategoriesCmd.class); + GuestOSCategoryVO guestOSCategory = Mockito.mock(GuestOSCategoryVO.class); + Filter filter = Mockito.mock(Filter.class); + Long id = 1L; + String name = "Ubuntu"; + String keyword = "Linux"; + Boolean featured = true; + Long zoneId = 1L; + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + Boolean isIso = true; + Boolean isVnf = false; + Mockito.when(listCmd.getId()).thenReturn(id); + Mockito.when(listCmd.getName()).thenReturn(name); + Mockito.when(listCmd.getKeyword()).thenReturn(keyword); + Mockito.when(listCmd.isFeatured()).thenReturn(featured); + Mockito.when(listCmd.getZoneId()).thenReturn(zoneId); + Mockito.when(listCmd.getArch()).thenReturn(arch); + Mockito.when(listCmd.isIso()).thenReturn(isIso); + Mockito.when(listCmd.isVnf()).thenReturn(isVnf); + SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); + Mockito.when(searchBuilder.entity()).thenReturn(guestOSCategory); + SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); + Mockito.when(_guestOSCategoryDao.createSearchBuilder()).thenReturn(searchBuilder); + Mockito.when(searchBuilder.create()).thenReturn(searchCriteria); + Mockito.when(templateDao.listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf)).thenReturn(Arrays.asList(1L, 2L)); + Pair, Integer> mockResult = new Pair<>(Arrays.asList(guestOSCategory), 1); + mockGuestOsJoin(); + Mockito.when(_guestOSCategoryDao.searchAndCount(Mockito.eq(searchCriteria), Mockito.any())).thenReturn(mockResult); + Pair, Integer> result = spy.listGuestOSCategoriesByCriteria(listCmd); + Assert.assertNotNull(result); + Assert.assertEquals(1, result.second().intValue()); + Assert.assertEquals(1, result.first().size()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).createSearchBuilder(); + Mockito.verify(templateDao, Mockito.times(1)).listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).searchAndCount(Mockito.eq(searchCriteria), Mockito.any()); + } + + @Test + public void testListGuestOSCategoriesByCriteria_NoResults() { + ListGuestOsCategoriesCmd listCmd = Mockito.mock(ListGuestOsCategoriesCmd.class); + GuestOSCategoryVO guestOSCategory = Mockito.mock(GuestOSCategoryVO.class); + Long id = 1L; + String name = "CentOS"; + String keyword = "Linux"; + Boolean featured = false; + Long zoneId = 1L; + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + Boolean isIso = false; + Boolean isVnf = false; + Mockito.when(listCmd.getId()).thenReturn(id); + Mockito.when(listCmd.getName()).thenReturn(name); + Mockito.when(listCmd.getKeyword()).thenReturn(keyword); + Mockito.when(listCmd.isFeatured()).thenReturn(featured); + Mockito.when(listCmd.getZoneId()).thenReturn(zoneId); + Mockito.when(listCmd.getArch()).thenReturn(arch); + Mockito.when(listCmd.isIso()).thenReturn(isIso); + Mockito.when(listCmd.isVnf()).thenReturn(isVnf); + SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); + Mockito.when(searchBuilder.entity()).thenReturn(guestOSCategory); + SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); + Mockito.when(_guestOSCategoryDao.createSearchBuilder()).thenReturn(searchBuilder); + Mockito.when(searchBuilder.create()).thenReturn(searchCriteria); + Mockito.when(templateDao.listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf)).thenReturn(Arrays.asList(1L, 2L)); + Pair, Integer> mockResult = new Pair<>(Arrays.asList(), 0); + Mockito.when(_guestOSCategoryDao.searchAndCount(Mockito.eq(searchCriteria), Mockito.any())).thenReturn(mockResult); + mockGuestOsJoin(); + Pair, Integer> result = spy.listGuestOSCategoriesByCriteria(listCmd); + Assert.assertNotNull(result); + Assert.assertEquals(0, result.second().intValue()); + Assert.assertEquals(0, result.first().size()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).createSearchBuilder(); + Mockito.verify(templateDao, Mockito.times(1)).listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).searchAndCount(Mockito.eq(searchCriteria), Mockito.any()); + } + + @Test + public void testListGuestOSCategoriesByCriteria_NoGuestOsIdsFound() { + ListGuestOsCategoriesCmd listCmd = Mockito.mock(ListGuestOsCategoriesCmd.class); + GuestOSCategoryVO guestOSCategory = Mockito.mock(GuestOSCategoryVO.class); + Long id = 1L; + String name = "Ubuntu"; + String keyword = "Linux"; + Boolean featured = true; + Long zoneId = 1L; + CPU.CPUArch arch = CPU.CPUArch.getDefault(); + Boolean isIso = true; + Boolean isVnf = false; + Mockito.when(listCmd.getId()).thenReturn(id); + Mockito.when(listCmd.getName()).thenReturn(name); + Mockito.when(listCmd.getKeyword()).thenReturn(keyword); + Mockito.when(listCmd.isFeatured()).thenReturn(featured); + Mockito.when(listCmd.getZoneId()).thenReturn(zoneId); + Mockito.when(listCmd.getArch()).thenReturn(arch); + Mockito.when(listCmd.isIso()).thenReturn(isIso); + Mockito.when(listCmd.isVnf()).thenReturn(isVnf); + SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); + Mockito.when(searchBuilder.entity()).thenReturn(guestOSCategory); + SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); + Mockito.when(_guestOSCategoryDao.createSearchBuilder()).thenReturn(searchBuilder); + Mockito.when(searchBuilder.create()).thenReturn(searchCriteria); + Mockito.when(templateDao.listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf)).thenReturn(Arrays.asList(1L, 2L)); + Pair, Integer> mockResult = new Pair<>(Arrays.asList(), 0); + when(_guestOSCategoryDao.searchAndCount(Mockito.eq(searchCriteria), Mockito.any())).thenReturn(mockResult); + mockGuestOsJoin(); + Pair, Integer> result = spy.listGuestOSCategoriesByCriteria(listCmd); + Assert.assertNotNull(result); + Assert.assertEquals(0, result.second().intValue()); + Assert.assertEquals(0, result.first().size()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).createSearchBuilder(); + Mockito.verify(templateDao, Mockito.times(1)).listTemplateIsoByArchVnfAndZone(zoneId, arch, isIso, isVnf); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).searchAndCount(Mockito.eq(searchCriteria), Mockito.any()); + } + + @Test + public void testListGuestOSCategoriesByCriteria_FilterById() { + ListGuestOsCategoriesCmd listCmd = Mockito.mock(ListGuestOsCategoriesCmd.class); + GuestOSCategoryVO guestOSCategory = Mockito.mock(GuestOSCategoryVO.class); + Long id = 1L; + Mockito.when(listCmd.getId()).thenReturn(id); + Mockito.when(listCmd.getZoneId()).thenReturn(null); + Mockito.when(listCmd.isIso()).thenReturn(null); + Mockito.when(listCmd.isVnf()).thenReturn(null); + SearchBuilder searchBuilder = Mockito.mock(SearchBuilder.class); + Mockito.when(searchBuilder.entity()).thenReturn(guestOSCategory); + SearchCriteria searchCriteria = Mockito.mock(SearchCriteria.class); + Mockito.when(_guestOSCategoryDao.createSearchBuilder()).thenReturn(searchBuilder); + Mockito.when(searchBuilder.create()).thenReturn(searchCriteria); + Pair, Integer> mockResult = new Pair<>(Arrays.asList(guestOSCategory), 1); + Mockito.when(_guestOSCategoryDao.searchAndCount(Mockito.eq(searchCriteria), Mockito.any())).thenReturn(mockResult); + mockGuestOsJoin(); + Pair, Integer> result = spy.listGuestOSCategoriesByCriteria(listCmd); + Assert.assertNotNull(result); + Assert.assertEquals(1, result.second().intValue()); + Assert.assertEquals(1, result.first().size()); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).createSearchBuilder(); + Mockito.verify(searchCriteria, Mockito.times(1)).setParameters("id", id); + Mockito.verify(_guestOSCategoryDao, Mockito.times(1)).searchAndCount(Mockito.eq(searchCriteria), Mockito.any()); + + } } diff --git a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java index 7f4344f30e4..7533767c00b 100644 --- a/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java +++ b/server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java @@ -1122,6 +1122,11 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches return null; } + @Override + public boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) { + return false; + } + @Override public void expungeLbVmRefs(List vmIds, Long batchSize) { } diff --git a/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java index 1b9923ad3ea..9cdcce8008e 100644 --- a/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/agent/lb/IndirectAgentLBServiceImplTest.java @@ -204,7 +204,7 @@ public class IndirectAgentLBServiceImplTest { public void testGetOrderedRunningHostIdsEmptyList() { doReturn(Collections.emptyList()).when(hostDao).findHostIdsByZoneClusterResourceStateTypeAndHypervisorType( Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); - Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID).isEmpty()); + Assert.assertTrue(agentMSLB.getOrderedHostIdList(DC_1_ID, false).isEmpty()); } @Test @@ -213,6 +213,6 @@ public class IndirectAgentLBServiceImplTest { .findHostIdsByZoneClusterResourceStateTypeAndHypervisorType(Mockito.eq(DC_1_ID), Mockito.eq(null), Mockito.eq(null), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); Assert.assertEquals(Arrays.asList(host1.getId(), host2.getId(), host3.getId(), host4.getId()), - agentMSLB.getOrderedHostIdList(DC_1_ID)); + agentMSLB.getOrderedHostIdList(DC_1_ID, false)); } } diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java index 9d80064d292..bc06d749078 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/formatinspector/Qcow2Inspector.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.storage.formatinspector; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -25,6 +26,7 @@ import org.apache.logging.log4j.Logger; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.math.BigInteger; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -108,6 +110,32 @@ public class Qcow2Inspector { return result; } + + /** + * Validates if the file has a minimum version. + * @param filePath Path of the file to be validated. + * @param minVersion the minimum version that it should contain. + * @throws RuntimeException If a IOException is thrown. + * @return true if file version is >= minVersion, false otherwise. + */ + public static boolean validateQcow2Version(String filePath, int minVersion) { + try (InputStream inputStream = new FileInputStream(filePath)) { + for (Qcow2HeaderField qcow2Header : Qcow2HeaderField.values()) { + if (qcow2Header != Qcow2HeaderField.VERSION) { + skipHeader(inputStream, qcow2Header, filePath); + continue; + } + + byte[] headerValue = readHeader(inputStream, qcow2Header, filePath); + int version = new BigInteger(headerValue).intValue(); + return version >= minVersion; + } + } catch (IOException ex) { + throw new CloudRuntimeException(String.format("Unable to validate file [%s] due to: ", filePath), ex); + } + return false; + } + /** * Skips the field's length in the InputStream. * @param qcow2InputStream InputStream of the QCOW2 being unraveled. diff --git a/setup/bindir/cloud-setup-databases.in b/setup/bindir/cloud-setup-databases.in index 5816ee1e9a9..e2d5b50fe6b 100755 --- a/setup/bindir/cloud-setup-databases.in +++ b/setup/bindir/cloud-setup-databases.in @@ -72,6 +72,7 @@ class DBDeployer(object): magicString = 'This_is_a_magic_string_i_think_no_one_will_duplicate' tmpMysqlFile = os.path.join(os.path.expanduser('~/'), 'cloudstackmysql.tmp.sql') mysqlBinPath = None + skipUsersAutoCreation = False def preRun(self): def backUpDbDotProperties(): @@ -219,6 +220,19 @@ for full help ""), ) + queriesToSkip = ( + ("CREATE USER cloud@`localhost` identified by 'cloud';", ""), + ("CREATE USER cloud@`%` identified by 'cloud';", ""), + ("GRANT ALL ON cloud.* to cloud@`localhost`;", ""), + ("GRANT ALL ON cloud.* to cloud@`%`;", ""), + ("GRANT ALL ON cloud_usage.* to cloud@`localhost`;", ""), + ("GRANT ALL ON cloud_usage.* to cloud@`%`;", ""), + ("GRANT process ON *.* TO cloud@`localhost`;", ""), + ("GRANT process ON *.* TO cloud@`%`;", ""), + ("DROP USER 'cloud'@'localhost' ;", "DO NULL;"), + ("DROP USER 'cloud'@'%' ;", "DO NULL;") + ) + scriptsToRun = ["create-database","create-schema", "create-database-premium","create-schema-premium"] if self.options.schemaonly: scriptsToRun = ["create-schema", "create-schema-premium"] @@ -227,6 +241,8 @@ for full help p = os.path.join(self.dbFilesPath,"%s.sql"%f) if not os.path.exists(p): continue text = open(p).read() + if self.options.skipUsersAutoCreation: + for t, r in queriesToSkip: text = text.replace(t,r) for t, r in replacements: text = text.replace(t,r) self.info("Applying %s"%p) self.runMysql(text, p, self.rootuser != None) @@ -472,6 +488,8 @@ for example: self.encryptionJarPath = self.options.encryptionJarPath if self.options.mysqlbinpath: self.mysqlBinPath = self.options.mysqlbinpath + if self.options.skipUsersAutoCreation: + self.skipUsersAutoCreation = self.options.skipUsersAutoCreation if self.options.encryptorVersion: self.encryptorVersion = "--encryptorversion %s" % self.options.encryptorVersion @@ -612,6 +630,9 @@ for example: self.parser.add_option("-g", "--encryptor-version", action="store", dest="encryptorVersion", default="V2", help="The encryptor version to be used to encrypt the values in db.properties") self.parser.add_option("-b", "--mysql-bin-path", action="store", dest="mysqlbinpath", help="The mysql installed bin path") + self.parser.add_option("-u", "--skip-users-auto-creation", action="store_true", dest="skipUsersAutoCreation", + help="Indicates whether to skip the auto-creation of users in the database. Use this flag when your database users " \ + "are already configured and you only want to populate the db.properties file.") (self.options, self.args) = self.parser.parse_args() parseCasualCredit() parseOtherOptions() diff --git a/setup/db/create-schema.sql b/setup/db/create-schema.sql index 3f14fccd010..10141185eb4 100755 --- a/setup/db/create-schema.sql +++ b/setup/db/create-schema.sql @@ -398,7 +398,7 @@ CREATE TABLE `cloud`.`op_lock` ( `waiters` int NOT NULL DEFAULT 0 COMMENT 'How many have the thread acquired this lock (reentrant)', PRIMARY KEY (`key`), INDEX `i_op_lock__mac_ip_thread`(`mac`, `ip`, `thread`) -) ENGINE=Memory DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`configuration` ( `category` varchar(255) NOT NULL DEFAULT 'Advanced', @@ -1793,7 +1793,7 @@ CREATE TABLE `cloud`.`op_nwgrp_work` ( INDEX `i_op_nwgrp_work__taken`(`taken`), INDEX `i_op_nwgrp_work__step`(`step`), INDEX `i_op_nwgrp_work__seq_no`(`seq_no`) -) ENGINE=MEMORY DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; CREATE TABLE `cloud`.`op_vm_ruleset_log` ( `id` bigint unsigned UNIQUE NOT NULL AUTO_INCREMENT COMMENT 'id', diff --git a/test/selenium/cspages/__init__.py b/systemvm/debian/opt/cloud/bin/cks_iso.sh similarity index 63% rename from test/selenium/cspages/__init__.py rename to systemvm/debian/opt/cloud/bin/cks_iso.sh index 13a83393a91..7cbcbc040d5 100644 --- a/test/selenium/cspages/__init__.py +++ b/systemvm/debian/opt/cloud/bin/cks_iso.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,3 +15,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +BASE_DIR=/var/www/html +CKS_ISO_DIR=$BASE_DIR/cks-iso +if [ "$1" == "true" ] +then + mkdir -p $CKS_ISO_DIR + echo "Options +Indexes" > $BASE_DIR/.htaccess + echo "Mounting CKS ISO into $CKS_ISO_DIR" + mount /dev/cdrom $CKS_ISO_DIR +else + echo "Unmounting CKS ISO from $CKS_ISO_DIR" + umount $CKS_ISO_DIR + echo "Options -Indexes" > $BASE_DIR/.htaccess + rm -rf $CKS_ISO_DIR +fi +echo "Restarting apache2 service" +service apache2 restart diff --git a/systemvm/debian/opt/cloud/bin/configure.py b/systemvm/debian/opt/cloud/bin/configure.py index cf0b71ab436..908f798a665 100755 --- a/systemvm/debian/opt/cloud/bin/configure.py +++ b/systemvm/debian/opt/cloud/bin/configure.py @@ -908,6 +908,7 @@ class CsVmMetadata(CsDataBag): if os.path.exists(metamanifest): fh = open(metamanifest, "a+") self.__exflock(fh) + fh.seek(0) if file not in fh.read(): fh.write(file + '\n') self.__unflock(fh) diff --git a/systemvm/debian/opt/cloud/bin/cs/CsConfig.py b/systemvm/debian/opt/cloud/bin/cs/CsConfig.py index a17f6ac4aa5..e6e738b042a 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsConfig.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsConfig.py @@ -114,6 +114,9 @@ class CsConfig(object): def expose_dns(self): return self.cmdline().idata().get('exposedns', 'false') == 'true' + def use_router_ip_as_resolver(self): + return self.cl.get_use_router_ip_as_resolver() + def get_dns(self): conf = self.cmdline().idata() dns = [] @@ -123,9 +126,10 @@ class CsConfig(object): else: dns.append(self.address().get_guest_ip()) - for name in ('dns1', 'dns2'): - if name in conf: - dns.append(conf[name]) + if 'userouteripresolver' not in conf: + for name in ('dns1', 'dns2'): + if name in conf: + dns.append(conf[name]) return dns def get_format(self): diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py b/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py index abbf23b4cdf..4c6b61240b8 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDatabag.py @@ -176,6 +176,11 @@ class CsCmdLine(CsDataBag): return self.idata()['useextdns'] return False + def get_use_router_ip_as_resolver(self): + if "userouteripresolver" in self.idata(): + return self.idata()['userouteripresolver'] + return False + def get_advert_int(self): if 'advert_int' in self.idata(): return self.idata()['advert_int'] diff --git a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py index 35849e4c5f8..1b34ee7e68d 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsDhcp.py @@ -110,7 +110,12 @@ class CsDhcp(CsDataBag): if gn.get_dns() and device: sline = "dhcp-option=tag:interface-%s-%s,6" % (device, idx) dns_list = [x for x in gn.get_dns() if x] - if self.config.is_dhcp() and not self.config.use_extdns(): + if (self.config.is_vpc() or self.config.is_router()) and ('is_vr_guest_gateway' in gn.data and gn.data['is_vr_guest_gateway']): + if gateway in dns_list: + dns_list.remove(gateway) + if gn.data['router_guest_ip'] != ip: + dns_list.insert(0, ip) + elif self.config.is_dhcp() and not self.config.use_extdns(): guest_ip = self.config.address().get_guest_ip() if guest_ip and guest_ip in dns_list and ip not in dns_list: # Replace the default guest IP in VR with the ip in additional IP ranges, if shared network has multiple IP ranges. @@ -141,9 +146,9 @@ class CsDhcp(CsDataBag): listen_address.append(gateway) listen_address.append(ip) # Add localized "data-server" records in /etc/hosts for VPC routers - if self.config.is_vpc() or self.config.is_router(): + if (self.config.is_vpc() or self.config.is_router()) and ('is_vr_guest_gateway' not in gn.data or (not gn.data['is_vr_guest_gateway'])): self.add_host(gateway, "%s data-server" % CsHelper.get_hostname()) - elif self.config.is_dhcp(): + elif self.config.is_dhcp() or (self.config.is_vpc() or self.config.is_router() and gn.data['is_vr_guest_gateway']) : self.add_host(ip, "%s data-server" % CsHelper.get_hostname()) idx += 1 diff --git a/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py index 615c61d98e3..65200fb8796 100755 --- a/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py +++ b/systemvm/debian/opt/cloud/bin/cs/CsGuestNetwork.py @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. from merge import DataBag -from . import CsHelper class CsGuestNetwork: @@ -39,6 +38,9 @@ class CsGuestNetwork: if not self.guest: return self.config.get_dns() + if self.config.use_router_ip_as_resolver(): + return [self.data['router_guest_ip']] + dns = [] if 'router_guest_gateway' in self.data and not self.config.use_extdns() and ('is_vr_guest_gateway' not in self.data or not self.data['is_vr_guest_gateway']): dns.append(self.data['router_guest_gateway']) diff --git a/test/integration/plugins/storpool/TestTagsOnStorPool.py b/test/integration/plugins/storpool/TestTagsOnStorPool.py index a66fb3570f4..ea5c2a4cc78 100644 --- a/test/integration/plugins/storpool/TestTagsOnStorPool.py +++ b/test/integration/plugins/storpool/TestTagsOnStorPool.py @@ -283,7 +283,7 @@ class TestStoragePool(cloudstackTestCase): virtualmachineid = self.virtual_machine.id, listall=True ) - self.vc_policy_tags(volumes, vm_tags, vm, True) + self.vc_policy_tags(volumes, vm_tags, vm, should_tags_exists=True) @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") @@ -323,7 +323,7 @@ class TestStoragePool(cloudstackTestCase): vm = list_virtual_machines(self.apiclient,id = self.virtual_machine.id, listall=True) vm_tags = vm[0].tags - self.vc_policy_tags(volumes, vm_tags, vm, True) + self.vc_policy_tags(volumes, vm_tags, vm, should_tags_exists=True) self.assertEqual(volume_attached.id, self.volume.id, "Is not the same volume ") @@ -455,7 +455,7 @@ class TestStoragePool(cloudstackTestCase): vm = list_virtual_machines(self.apiclient,id = self.virtual_machine.id, listall=True) vm_tags = vm[0].tags - self.vc_policy_tags(volumes, vm_tags, vm, True) + self.vc_policy_tags(volumes, vm_tags, vm, should_tags_exists=True) self.assertEqual( self.random_data_0, @@ -491,14 +491,12 @@ class TestStoragePool(cloudstackTestCase): list_snapshot_response = VmSnapshot.list( self.apiclient, - #vmid=self.virtual_machine.id, virtualmachineid=self.virtual_machine.id, listall=False) self.debug('list_snapshot_response -------------------- %s' % list_snapshot_response) self.assertIsNone(list_snapshot_response, "snapshot is already deleted") - @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") def test_06_remove_vcpolicy_tag_when_disk_detached(self): """ Test remove vc-policy tag to disk detached from VM""" @@ -513,7 +511,7 @@ class TestStoragePool(cloudstackTestCase): self.apiclient, self.volume_2 ) - self.vc_policy_tags( volumes, vm_tags, vm, False) + self.vc_policy_tags( volumes, vm_tags, vm, should_tags_exists=False) @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") def test_07_delete_vcpolicy_tag(self): @@ -550,7 +548,7 @@ class TestStoragePool(cloudstackTestCase): virtualmachineid = self.virtual_machine2.id, listall=True, type = "ROOT" ) - self.vc_policy_tags(volume, vm_tags, vm, True) + self.vc_policy_tags(volume, vm_tags, vm, should_tags_exists=True) snapshot = Snapshot.create( self.apiclient, @@ -571,8 +569,8 @@ class TestStoragePool(cloudstackTestCase): vm = list_virtual_machines(self.apiclient,id = self.virtual_machine2.id) vm_tags = vm[0].tags - vol = list_volumes(self.apiclient, id = snapshot.volumeid, listall=True) - self.vc_policy_tags(vol, vm_tags, vm, True) + vol = list_volumes(self.apiclient, id=snapshot.volumeid, listall=True) + self.vc_policy_tags(vol, vm_tags, vm, should_tags_exists=True) @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") def test_09_remove_vm_tags_on_datadisks_attached_to_destroyed_vm(self): @@ -586,38 +584,82 @@ class TestStoragePool(cloudstackTestCase): vm_tags = vm[0].tags volumes = list_volumes( self.apiclient, - virtualmachineid = self.virtual_machine3.id, listall=True + virtualmachineid=self.virtual_machine3.id, listall=True ) - self.vc_policy_tags(volumes, vm_tags, vm, True) + self.vc_policy_tags(volumes, vm_tags, vm, should_tags_exists=True) volumes = list_volumes( self.apiclient, - virtualmachineid = self.virtual_machine3.id, listall=True, type="DATADISK" + virtualmachineid=self.virtual_machine3.id, listall=True, type="DATADISK" ) self.virtual_machine3.delete(self.apiclient, expunge=True) - self.vc_policy_tags(volumes, vm_tags, vm, False) + self.vc_policy_tags(volumes, vm_tags, vm, should_tags_exists=False) - def vc_policy_tags(self, volumes, vm_tags, vm, should_tags_exists=None): - vcPolicyTag = False - cvmTag = False + @attr(tags=["advanced", "advancedns", "smoke"], required_hardware="true") + def test_10_check_tags_on_deployed_vm_with_data_disk(self): + """ + Check disk and cvm tags are set on all volumes when VM is deployed with additional DATA disk + Detach the DATA disk + """ + vm = VirtualMachine.create( + self.apiclient, + {"name":"StorPool-%s" % uuid.uuid4() }, + zoneid=self.zone.id, + templateid=self.template.id, + accountid=self.account.name, + domainid=self.account.domainid, + serviceofferingid=self.service_offering.id, + hypervisor=self.hypervisor, + diskofferingid=self.disk_offerings.id, + size=2, + rootdisksize=10 + ) + volumes = list_volumes( + self.apiclient, + virtualmachineid=vm.id, listall=True + ) + vm1 = list_virtual_machines(self.apiclient,id=vm.id, listall=True) + vm_tags = vm1[0].tags + self.vc_policy_tags(volumes, vm_tags, vm1, False, True) + vm.stop(self.apiclient, forced=True) + volumes = list_volumes( + self.apiclient, + virtualmachineid=vm.id, listall=True, type="DATADISK" + ) + + self.debug("detaching volume %s" % volumes) + VirtualMachine.detach_volume(vm, self.apiclient, volumes[0]) + self.vc_policy_tags(volumes, vm_tags, vm1, False, False) + + def vc_policy_tags(self, volumes, vm_tags, vm, tag_check=True, should_tags_exists=None,): + vc_policy_tag = False + cvm_tag = False + disk_id_tag = False for v in volumes: name = v.path.split("/")[3] - spvolume = self.spapi.volumeList(volumeName="~" + name) - tags = spvolume[0].tags + volume = self.spapi.volumeList(volumeName="~" + name) + tags = volume[0].tags + self.debug("Tags %s" % tags) for t in tags: for vm_tag in vm_tags: if t == vm_tag.key: - vcPolicyTag = True + vc_policy_tag = True self.assertEqual(tags[t], vm_tag.value, "Tags are not equal") - if t == 'cvm': - cvmTag = True - self.assertEqual(tags[t], vm[0].id, "CVM tag is not the same as vm UUID") - #self.assertEqual(tag.tags., second, msg) + if t == 'cvm': + cvm_tag = True + self.assertEqual(tags[t], vm[0].id, "CVM tag is not the same as vm UUID") + if t == 'disk': + disk_id_tag = True + self.assertEqual(tags[t], str(v.deviceid), "Disk tag is not equal to the device ID") if should_tags_exists: - self.assertTrue(vcPolicyTag, "There aren't volumes with vm tags") - self.assertTrue(cvmTag, "There aren't volumes with vm tags") + if tag_check: + self.assertTrue(vc_policy_tag, "There aren't volumes with vc policy tags") + self.assertTrue(cvm_tag, "There aren't volumes with vm UUID tags") + self.assertTrue(disk_id_tag, "There aren't volumes with vm disk tag") else: - self.assertFalse(vcPolicyTag, "The tags should be removed") - self.assertFalse(cvmTag, "The tags should be removed") + if tag_check: + self.assertFalse(vc_policy_tag, "The vc policy tag should be removed") + self.assertFalse(cvm_tag, "The cvm tag should be removed") + self.assertFalse(disk_id_tag, "The disk tag should be removed") diff --git a/test/integration/smoke/test_kubernetes_clusters.py b/test/integration/smoke/test_kubernetes_clusters.py index 20f1cb3224a..a77829bc255 100644 --- a/test/integration/smoke/test_kubernetes_clusters.py +++ b/test/integration/smoke/test_kubernetes_clusters.py @@ -35,7 +35,9 @@ from marvin.cloudstackAPI import (listInfrastructure, destroyVirtualMachine, deleteNetwork, addVirtualMachinesToKubernetesCluster, - removeVirtualMachinesFromKubernetesCluster) + removeVirtualMachinesFromKubernetesCluster, + addNodesToKubernetesCluster, + removeNodesFromKubernetesCluster) from marvin.cloudstackException import CloudstackAPIException from marvin.codes import PASS, FAILED from marvin.lib.base import (Template, @@ -49,28 +51,84 @@ from marvin.lib.base import (Template, VPC, NetworkACLList, NetworkACL, - VirtualMachine) + VirtualMachine, + PublicIPAddress, + FireWallRule, + NATRule) from marvin.lib.utils import (cleanup_resources, validateList, random_gen) from marvin.lib.common import (get_zone, get_domain, - get_template) + get_template, + get_test_template) from marvin.sshClient import SshClient from nose.plugins.attrib import attr from marvin.lib.decoratorGenerators import skipTestIf from kubernetes import client, config -import time, io, yaml +import time, io, yaml, random _multiprocess_shared_ = True k8s_cluster = None +k8s_cluster_node_offerings = None VPC_DATA = { "cidr": "10.1.0.0/22", "tier1_gateway": "10.1.1.1", "tier_netmask": "255.255.255.0" } +RAND_SUFFIX = random_gen() +NODES_TEMPLATE = { + "kvm": { + "name": "cks-u2204-kvm-" + RAND_SUFFIX, + "displaytext": "cks-u2204-kvm-" + RAND_SUFFIX, + "format": "qcow2", + "hypervisor": "kvm", + "ostype": "Ubuntu 22.04 LTS", + "url": "https://download.cloudstack.org/testing/custom_templates/ubuntu/22.04/cks-ubuntu-2204-kvm.qcow2.bz2", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True", + "forcks": "True" + }, + "xenserver": { + "name": "cks-u2204-hyperv-" + RAND_SUFFIX, + "displaytext": "cks-u2204-hyperv-" + RAND_SUFFIX, + "format": "vhd", + "hypervisor": "xenserver", + "ostype": "Ubuntu 22.04 LTS", + "url": "https://download.cloudstack.org/testing/custom_templates/ubuntu/22.04/cks-ubuntu-2204-hyperv.vhd.zip", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True", + "forcks": "True" + }, + "hyperv": { + "name": "cks-u2204-hyperv-" + RAND_SUFFIX, + "displaytext": "cks-u2204-hyperv-" + RAND_SUFFIX, + "format": "vhd", + "hypervisor": "hyperv", + "ostype": "Ubuntu 22.04 LTS", + "url": "https://download.cloudstack.org/testing/custom_templates/ubuntu/22.04/cks-ubuntu-2204-hyperv.vhd.zip", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True", + "forcks": "True" + }, + "vmware": { + "name": "cks-u2204-vmware-" + RAND_SUFFIX, + "displaytext": "cks-u2204-vmware-" + RAND_SUFFIX, + "format": "ova", + "hypervisor": "vmware", + "ostype": "Ubuntu 22.04 LTS", + "url": "https://download.cloudstack.org/testing/custom_templates/ubuntu/22.04/cks-ubuntu-2204-vmware.ova", + "requireshvm": "True", + "ispublic": "True", + "isextractable": "True", + "forcks": "True" + } +} class TestKubernetesCluster(cloudstackTestCase): @@ -84,6 +142,7 @@ class TestKubernetesCluster(cloudstackTestCase): cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ cls.hypervisorNotSupported = False + cls.hypervisorIsNotVmware = cls.hypervisor.lower() != "vmware" if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: cls.hypervisorNotSupported = True cls.setup_failed = False @@ -129,13 +188,40 @@ class TestKubernetesCluster(cloudstackTestCase): (cls.services["cks_kubernetes_versions"][cls.k8s_version_to]["semanticversion"], cls.services["cks_kubernetes_versions"][cls.k8s_version_to]["url"], e)) if cls.setup_failed == False: + cls.nodes_template = None + cls.mgmtSshKey = None + if cls.hypervisor.lower() == "vmware": + cls.nodes_template = get_test_template(cls.apiclient, + cls.zone.id, + cls.hypervisor, + NODES_TEMPLATE) + cls.nodes_template.update(cls.apiclient, forcks=True) + cls.mgmtSshKey = cls.getMgmtSshKey() cks_offering_data = cls.services["cks_service_offering"] cks_offering_data["name"] = 'CKS-Instance-' + random_gen() cls.cks_service_offering = ServiceOffering.create( cls.apiclient, cks_offering_data ) + cks_offering_data["name"] = 'CKS-Worker-Offering-' + random_gen() + cls.cks_worker_nodes_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + cks_offering_data["name"] = 'CKS-Control-Offering-' + random_gen() + cls.cks_control_nodes_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + cks_offering_data["name"] = 'CKS-Etcd-Offering-' + random_gen() + cls.cks_etcd_nodes_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) cls._cleanup.append(cls.cks_service_offering) + cls._cleanup.append(cls.cks_worker_nodes_offering) + cls._cleanup.append(cls.cks_control_nodes_offering) + cls._cleanup.append(cls.cks_etcd_nodes_offering) cls.domain = get_domain(cls.apiclient) cls.account = Account.create( cls.apiclient, @@ -204,6 +290,19 @@ class TestKubernetesCluster(cloudstackTestCase): name="vmware.create.full.clone", value=value) + @classmethod + def getMgmtSshKey(cls): + """Get the management server SSH public key""" + sshClient = SshClient( + cls.mgtSvrDetails["mgtSvrIp"], + 22, + cls.mgtSvrDetails["user"], + cls.mgtSvrDetails["passwd"] + ) + command = "cat /var/cloudstack/management/.ssh/id_rsa.pub" + response = sshClient.execute(command) + return str(response[0]) + @classmethod def restartServer(cls): """Restart management server""" @@ -644,6 +743,151 @@ class TestKubernetesCluster(cloudstackTestCase): self.deleteKubernetesClusterAndVerify(cluster.id) return + @attr(tags=["advanced", "smoke"], required_hardware="true") + def test_12_test_deploy_cluster_different_offerings_per_node_type(self): + """Test creating a CKS cluster with different offerings per node type + + # Validate the following on Kubernetes cluster creation: + # - Use a service offering for control nodes + # - Use a service offering for worker nodes + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + cluster = self.getValidKubernetesCluster(worker_offering=self.cks_worker_nodes_offering, + control_offering=self.cks_control_nodes_offering) + self.assertEqual( + cluster.workerofferingid, + self.cks_worker_nodes_offering.id, + "Check Worker Nodes Offering {}, {}".format(cluster.workerofferingid, self.cks_worker_nodes_offering.id) + ) + self.assertEqual( + cluster.controlofferingid, + self.cks_control_nodes_offering.id, + "Check Control Nodes Offering {}, {}".format(cluster.workerofferingid, self.cks_worker_nodes_offering.id) + ) + self.assertEqual( + cluster.etcdnodes, + 0, + "No Etcd Nodes expected but got {}".format(cluster.etcdnodes) + ) + self.debug("Deleting Kubernetes cluster with ID: %s" % cluster.id) + self.deleteKubernetesClusterAndVerify(cluster.id) + return + + @attr(tags=["advanced", "smoke"], required_hardware="true") + @skipTestIf("hypervisorIsNotVmware") + def test_13_test_add_external_nodes_to_cluster(self): + """Test adding and removing external nodes to CKS clusters + + # Validate the following: + # - Deploy Kubernetes Cluster + # - Deploy VM on the same network as the Kubernetes cluster with the worker nodes offering and CKS ready template + # - Add external node to the Kubernetes Cluster + # - Remove external node from the Kubernetes Cluster + """ + if self.setup_failed == True: + self.fail("Setup incomplete") + cluster = self.getValidKubernetesCluster(worker_offering=self.cks_worker_nodes_offering, + control_offering=self.cks_control_nodes_offering) + self.assertEqual( + cluster.size, + 1, + "Expected 1 worker node but got {}".format(cluster.size) + ) + self.services["virtual_machine"]["template"] = self.nodes_template.id + external_node = VirtualMachine.create(self.apiclient, + self.services["virtual_machine"], + zoneid=self.zone.id, + accountid=self.account.name, + domainid=self.account.domainid, + rootdiskcontroller="osdefault", + rootdisksize=8, + serviceofferingid=self.cks_worker_nodes_offering.id, + networkids=cluster.networkid) + + # Acquire public IP and create Port Forwarding Rule and Firewall rule for SSH access + free_ip_addresses = PublicIPAddress.list( + self.apiclient, + domainid=self.account.domainid, + account=self.account.name, + forvirtualnetwork=True, + state='Free' + ) + random.shuffle(free_ip_addresses) + external_node_ip = free_ip_addresses[0] + external_node_ipaddress = PublicIPAddress.create( + self.apiclient, + zoneid=self.zone.id, + networkid=cluster.networkid, + ipaddress=external_node_ip.ipaddress + ) + self.debug("Creating Firewall rule for VM ID: %s" % external_node.id) + fw_rule = FireWallRule.create( + self.apiclient, + ipaddressid=external_node_ip.id, + protocol='TCP', + cidrlist=['0.0.0.0/0'], + startport=22, + endport=22 + ) + pf_rule = { + "privateport": 22, + "publicport": 22, + "protocol": "TCP" + } + nat_rule = NATRule.create( + self.apiclient, + external_node, + pf_rule, + ipaddressid=external_node_ip.id + ) + + # Add the management server SSH key to the authorized hosts on the external node + node_ssh_client = SshClient( + external_node_ip.ipaddress, + 22, + 'cloud', + 'cloud', + retries=30, + delay=10 + ) + node_ssh_client.execute("echo '" + self.mgmtSshKey + "' > ~/.ssh/authorized_keys") + # Remove acquired public IP address and rules + nat_rule.delete(self.apiclient) + fw_rule.delete(self.apiclient) + external_node_ipaddress.delete(self.apiclient) + + self.addExternalNodesToKubernetesCluster(cluster.id, [external_node.id]) + cluster = self.listKubernetesCluster(cluster.id) + self.assertEqual( + cluster.size, + 2, + "Expected 2 worker nodes but got {}".format(cluster.size) + ) + self.removeExternalNodesFromKubernetesCluster(cluster.id, [external_node.id]) + cluster = self.listKubernetesCluster(cluster.id) + self.assertEqual( + cluster.size, + 1, + "Expected 1 worker node but got {}".format(cluster.size) + ) + VirtualMachine.delete(external_node, self.apiclient, expunge=True) + self.debug("Deleting Kubernetes cluster with ID: %s" % cluster.id) + self.deleteKubernetesClusterAndVerify(cluster.id) + return + + def addExternalNodesToKubernetesCluster(self, cluster_id, vm_list): + cmd = addNodesToKubernetesCluster.addNodesToKubernetesClusterCmd() + cmd.id = cluster_id + cmd.nodeids = vm_list + return self.apiclient.addNodesToKubernetesCluster(cmd) + + def removeExternalNodesFromKubernetesCluster(self, cluster_id, vm_list): + cmd = removeNodesFromKubernetesCluster.removeNodesFromKubernetesClusterCmd() + cmd.id = cluster_id + cmd.nodeids = vm_list + return self.apiclient.removeNodesFromKubernetesCluster(cmd) + def addVirtualMachinesToKubernetesCluster(self, cluster_id, vm_list): cmd = addVirtualMachinesToKubernetesCluster.addVirtualMachinesToKubernetesClusterCmd() cmd.id = cluster_id @@ -658,8 +902,8 @@ class TestKubernetesCluster(cloudstackTestCase): return self.apiclient.removeVirtualMachinesFromKubernetesCluster(cmd) - - def createKubernetesCluster(self, name, version_id, size=1, control_nodes=1, cluster_type='CloudManaged'): + def createKubernetesCluster(self, name, version_id, size=1, control_nodes=1, etcd_nodes=0, cluster_type='CloudManaged', + workers_offering=None, control_offering=None, etcd_offering=None): createKubernetesClusterCmd = createKubernetesCluster.createKubernetesClusterCmd() createKubernetesClusterCmd.name = name createKubernetesClusterCmd.description = name + "-description" @@ -672,6 +916,22 @@ class TestKubernetesCluster(cloudstackTestCase): createKubernetesClusterCmd.account = self.account.name createKubernetesClusterCmd.domainid = self.domain.id createKubernetesClusterCmd.clustertype = cluster_type + if workers_offering: + createKubernetesClusterCmd.nodeofferings.append({ + "node": "WORKER", + "offering": workers_offering.id + }) + if control_offering: + createKubernetesClusterCmd.nodeofferings.append({ + "node": "CONTROL", + "offering": control_offering.id + }) + if etcd_nodes > 0 and etcd_offering: + createKubernetesClusterCmd.etcdnodes = etcd_nodes + createKubernetesClusterCmd.nodeofferings.append({ + "node": "ETCD", + "offering": etcd_offering.id + }) if self.default_network: createKubernetesClusterCmd.networkid = self.default_network.id clusterResponse = self.apiclient.createKubernetesCluster(createKubernetesClusterCmd) @@ -735,7 +995,8 @@ class TestKubernetesCluster(cloudstackTestCase): retries = retries - 1 return False - def getValidKubernetesCluster(self, size=1, control_nodes=1, version={}): + def getValidKubernetesCluster(self, size=1, control_nodes=1, version={}, etcd_nodes=0, + worker_offering=None, control_offering=None, etcd_offering=None): cluster = k8s_cluster # Does a cluster already exist ? @@ -743,7 +1004,9 @@ class TestKubernetesCluster(cloudstackTestCase): if not version: version = self.kubernetes_version_v2 self.debug("No existing cluster available, k8s_cluster: %s" % cluster) - return self.createNewKubernetesCluster(version, size, control_nodes) + return self.createNewKubernetesCluster(version, size, control_nodes, etcd_nodes=etcd_nodes, + worker_offering=worker_offering, control_offering=control_offering, + etcd_offering=etcd_offering) # Is the existing cluster what is needed ? valid = cluster.size == size and cluster.controlnodes == control_nodes @@ -759,7 +1022,9 @@ class TestKubernetesCluster(cloudstackTestCase): if cluster == None: # Looks like the cluster disappeared ! self.debug("Existing cluster, k8s_cluster ID: %s not returned by list API" % cluster_id) - return self.createNewKubernetesCluster(version, size, control_nodes) + return self.createNewKubernetesCluster(version, size, control_nodes, etcd_nodes=etcd_nodes, + worker_offering=worker_offering, control_offering=control_offering, + etcd_offering=etcd_offering) if valid: try: @@ -775,13 +1040,18 @@ class TestKubernetesCluster(cloudstackTestCase): self.deleteKubernetesClusterAndVerify(cluster.id, False, True) self.debug("No valid cluster, need to deploy a new one") - return self.createNewKubernetesCluster(version, size, control_nodes) + return self.createNewKubernetesCluster(version, size, control_nodes, etcd_nodes=etcd_nodes, + worker_offering=worker_offering, control_offering=control_offering, + etcd_offering=etcd_offering) - def createNewKubernetesCluster(self, version, size, control_nodes) : + def createNewKubernetesCluster(self, version, size, control_nodes, etcd_nodes=0, + worker_offering=None, control_offering=None, etcd_offering=None): name = 'testcluster-' + random_gen() self.debug("Creating for Kubernetes cluster with name %s" % name) try: - cluster = self.createKubernetesCluster(name, version.id, size, control_nodes) + cluster = self.createKubernetesCluster(name, version.id, size, control_nodes, etcd_nodes=etcd_nodes, + workers_offering=worker_offering, control_offering=control_offering, + etcd_offering=etcd_offering) self.verifyKubernetesCluster(cluster, name, version.id, size, control_nodes) except Exception as ex: cluster = self.listKubernetesCluster(cluster_name = name) diff --git a/test/pom.xml b/test/pom.xml index 3825f07d8c5..ec23feac70d 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -44,16 +44,6 @@ org.jenkins-ci trilead-ssh2 - - org.openqa.selenium.server - selenium-server - ${cs.selenium.server.version} - - - org.seleniumhq.selenium.client-drivers - selenium-java-client-driver - ${cs.selenium-java-client-driver.version} - compile diff --git a/test/selenium/ReadMe.txt b/test/selenium/ReadMe.txt deleted file mode 100644 index bc968f1dc93..00000000000 --- a/test/selenium/ReadMe.txt +++ /dev/null @@ -1,66 +0,0 @@ -############################################## - -Questions? Post'em @ dev@cloudstack.apache.org - -############################################## - -This files contains following: - -1) Installation requirements -2) Testing pre-requisites -3) Running the Tests and Generating the report -############################################## - - - -########################################################################################################################################## - -1) Installation Requirements ---------------------------- - - -1) Firefox depending on your OS (Good to have Firebug and Selenium IDE for troubleshooting and dev work) - - -2) Install Python 2.7. - - -3) Now Open CMD/Terminal and type all of following - -- pip install pycrypto (Installs Pycrypto) -- pip install paramiko (Install paramiko) -- pip install unittest-xml-reporting (Install XML Test Runner) -- pip install -U selenium (Installs Selenium) - -4) Get PhoantomJS for your OS from http://phantomjs.org/ - -- PhantomJS will run selenium test in headless mode. Follow the instruction on PhantomJS.org. -- Make sure the executable is in PATH. (TIP: Drop it in Python27 folder :-)) - -5) Now get the HTMLTestRunner for nice looking report generation. -- http://tungwaiyip.info/software/HTMLTestRunner.html -- Download and put this file into Lib of your python installation. - - -########################################################################################################################################## - -2) Test Prerequisites ---------------------- - -- Download and install CS. /cwiki.apache.org has links to Installation Guide and API reference. -- Log into the management server and Add a Zone. (Must be Advance Zone and Hypervisor type must be Xen) - - -########################################################################################################################################## - -3) Running the Test and Generating the report ---------------------------------------------- - -- Folder smoke contains main.py -- main.py is the file where all the tests are serialized. -- main.py supports HTML and XML reporting. Please refer to end of file to choose either. -- Typical usage is: python main.py 10.1.1.10 >> result.xml for XML Reporting -- And python main.py 10.1.1.10 >> result.html for HTML Reporting. -- 10.1.1.10 (your management server IP) is an argument required for main. - -########################################################################################################################################## diff --git a/test/selenium/browser/firefox.py b/test/selenium/browser/firefox.py deleted file mode 100644 index e4e8e1c6262..00000000000 --- a/test/selenium/browser/firefox.py +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from selenium import webdriver -import time -from selenium.common.exceptions import WebDriverException -from selenium.common.exceptions import NoSuchElementException - -class Firefox(object): - def __init__(self, x_pos = 0, y_pos = 0, x_size = 1024, y_size = 768, timeout = 30): - self.browser = None - self.browser = webdriver.Firefox() - self.browser.set_page_load_timeout(timeout) - self.browser.set_window_position(x_pos, y_pos) - self.browser.set_window_size(x_size, y_size) - - def get_browser(self): - return self.browser - - def set_url(self, url): - if url == None or url == "": - print "A valid url is required" - return - self.url = url - self.browser.get(url) - - def quit_browser(self): - try: - self.browser.quit() - except NoSuchElementException as err: - print "Element error({0})".format(err.msg) - except WebDriverException as err: - print "WebDriver error({0})".format(err.msg) - - -if __name__ == "__main__": - # Create a new instance of the Firefox driver - browser = Firefox("Firefox") - browser.set_url("http://10.88.90.84:8080/client/") - time.sleep(3) - browser.quit_browser() diff --git a/test/selenium/common/Global_Locators.py b/test/selenium/common/Global_Locators.py deleted file mode 100644 index 0219f3b2e7b..00000000000 --- a/test/selenium/common/Global_Locators.py +++ /dev/null @@ -1,230 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' -Variable Names are as follows -Logical Page Descriptor_____What Element Represents and/or where it is_____LocatorType - - -For Example :: - -instances_xpath = "//div[@id='navigation']/ul/li[2]/span[2]" - -Means this is:: xpath link for Instances which is present on Dashboard. -Any test cases that requires to go into Instances from Dashboard can use this variable now. - -This may not be intuitive as you go deep into the tree. - - - -for example - -stopinstanceforce_id - -The best way to know what this represents is to track by variable name -Under Instances / any instance is click on any instance (applies to any instance) / stop instance has a force stop check box when you click. -This link represents that. - - -Steps below do not have global locators. - -PF rule steps including and after filling port numbers. (Refer to vmLifeAndNetwork.py / def test_PF) -FW rule steps including and after filling port numbers. (Refer to vmLifeAndNetwork.py / def test_PF) -ADD Disk Offering page has Names, description, storage type etc etc -ADD Compute Offering page has Names, description, CPU Cores, CPU clocks type etc etc - -Create Acc, Delete Acc, Login and Logout are for test flow and are not test cases. They do not have global Locators. - -Such and many more data entry points that appear only once and hence we do not need glonal names for them. They are hard coded as and when needed in the scripts. - - -''' - -################################################################################################################################################################################################ - -## Links on the Main UI page (Dash board). Listed in the order they appear on screen -dashboard_xpath = "//div[@id='navigation']/ul/li" -instances_xpath = "//div[@id='navigation']/ul/li[2]/span[2]" # Link for Instance and following as self explanatory -storage_xpath = "//div[@id='navigation']/ul/li[3]/span[2]" -network_xpath = "//div[@id='navigation']/ul/li[4]/span[2]" -templates_xpath = "//div[@id='navigation']/ul/li[5]/span[2]" -events_xpath = "//div[@id='navigation']/ul/li[6]/span[2]" -projects_xpath = "//div[@id='navigation']/ul/li[7]/span[2]" -accounts_xpath = "//div[@id='navigation']/ul/li[8]/span[2]" -domains_xpath = "//div[@id='navigation']/ul/li[9]/span[2]" -infrastructure_xpath = "//div[@id='navigation']/ul/li[10]/span[2]" -globalSettings_xpath = "//div[@id='navigation']/ul/li[11]/span[2]" -serviceOfferings_xpath = "//div[@id='navigation']/ul/li[12]/span[2]" - -################################################################################################################################################################################################ - -## Instances Page -## Instances Main page - - -# Add Instance Button on top right corner of Instances page -add_instance_xpath = "//div[2]/div/div[2]/div/div[2]/span" - -# Add Instance Wizard next button -add_instance_next_xpath = "//div[4]/div[2]/div[3]/div[3]/span" - -# Table that lists all VM's under Instances page; General usage is to traverse through this table and search for the VM we are interested in. -instances_table_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div[2]/table/tbody/tr/td/span" - - -# Click any instance and following are available - -# Click ok on confirmation pop-up box for most actions listed below -actionconfirm_xpath = ("//button[@type='button']") - -# status of VM running. Click on VM > 3rd row in table -state_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div[2]/div[2]/div/div/div[2]/div/table/tbody/tr[3]/td[2]/span" - -# Stop instance icon -stopinstance_css = "a[alt=\"Stop Instance\"] > span.icon" - -# stop instance forcefully check box available after stop instance is executed in separate pop up -stopinstanceforce_id = ("force_stop") - -# start instance icon -startinstance_css = "a[alt=\"Start Instance\"] > span.icon" - -yesconfirmation_xapth = "(//button[@type='button'])[2]" - - -# Destroy instance icon -destroyinstance_css = "a[alt=\"Destroy Instance\"] > span.icon" - -#Restore Instance icon -restoreinstance_css = "a[alt=\"Restore Instance\"] > span.icon" - -# Reboot instance -rebootinstance_css = "a[alt=\"Reboot Instance\"] > span.icon" - -################################################################################################################################################################################################ - - -## Network Page - -# Table that lists all Networks under Network page; General usage is to traverse through this table and search for the network we are interested in. -network_networktable_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div[3]/div[2]/div/div[2]/table/tbody/tr/td/span" - -# View IP addresses button on each network page -viewIp_css="div.view-all > a > span" - -# Acquire a new ip -acquireIP_xpath="//div[2]/div/div/div[2]/span" -# List of IP's within a netork table -network_iptables_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div[3]/div[2]/div/div[2]/table/tbody/tr/td/span" -# Configuration tab for each IP -ipConfiguration_text="Configuration" -# PF under configuration for each IP -ip_PF = "li.portForwarding > div.view-details" - - -################################################################################################################################################################################################ - - -## Servivce Offering Page - -# Selects Compute offering from drop down menu -Offering_compute_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[1]" - -# Selects System offering from drop down menu -Offering_system_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[2]" - -# Selects Disk offering from drop down menu -Offering_disk_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[3]" - -# Selects Network offering from drop down menu -Offering_network_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[4]" - -# Add Offering -Offering_add_xpath ="//div[3]/span" - -# Points to tbale that lists Offerings -Offering_table_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div[2]/table/tbody/tr/td/span" - -# Edit Button -Offering_edit_css = "a[alt=\"Edit\"] > span.icon" - -# Edit name box -Offering_editname_name = "name" - -# Edit description box -Offering_editdescription_name = "displaytext" - -# Edit finished click ok -Offering_editdone_css="div.button.done" - -# delete offering button for Disk only -Offering_delete_css = "a[alt=\"Delete Disk Offering\"] > span.icon" - -# delete offering button for Compute only -Offering_deletecompute_css = "a[alt=\"Delete Service Offering\"] > span.icon" - - - - -################################################################################################################################################################################################ - - -#### Templates Page - -# Selects Templates from drop down -template_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[1]" - -# Selects ISO from drop down -iso_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[2]" - -# Add Template -AddTemplate_xpath = "//div[3]/span" - -# Points to table where all templates are -template_table_xpath ="/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div[2]/table/tbody/tr/td/span" - -# Edit Template Button -template_edit_css = "a[alt=\"Edit\"] > span.icon" - -# Edit finished click OK -template_editdone_css = "div.button.done" - -# Delete Template button -template_delete_css = "a[alt=\"Delete Template\"] > span.icon" - - -################################################################################################################################################################################################ - - -## Login Page - -# Username box -login_username_css = "body.login > div.login > form > div.fields > div.field.username > input[name=\"username\"]" # Login>Username TextBox - -# Password Box -login_password_css = "body.login > div.login > form > div.fields > div.field.password > input[name=\"password\"]" # LoginPassword TextBox - -# Click ok to login -login_submit_css = "body.login > div.login > form > div.fields > input[type=\"submit\"]" # Login>Login Button (Submit button) - - -################################################################################################################################################################################################ - - -## Logout -logout_css = "div.cloudstack3-container > div.container > div.header > div.controls div.user > div.user-options > a[text=\"Logout\"]" # Logout -# logout_css = "div#header > div#user-options > a[href='#']" diff --git a/test/selenium/common/shared.py b/test/selenium/common/shared.py deleted file mode 100644 index 21c598ebbee..00000000000 --- a/test/selenium/common/shared.py +++ /dev/null @@ -1,148 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#!/usr/bin/python -# coding: latin-1 - -from selenium.selenium import selenium -from selenium.common.exceptions import NoSuchElementException -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.common.exceptions import WebDriverException -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.support import expected_conditions as EC # available since 2.26.0 -from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0 -import re, sys, time, traceback - -def try_except_decor(func): - def try_except(*args, **kwargs): - try: - return func(*args, **kwargs) - except WebDriverException as err: - exc_type, exc_value, exc_traceback = sys.exc_info() - print "WebDriver error. Function: {0}, error: {1}".format(func.func_code, err) - print repr(traceback.format_exception(exc_type, exc_value,exc_traceback)) - except NoSuchElementException as err: - exc_type, exc_value, exc_traceback = sys.exc_info() - print "Element error. Function: {0}, error: {1}".format(func.func_code, err) - print repr(traceback.format_exception(exc_type, exc_value,exc_traceback)) - except TimeoutException as err: - exc_type, exc_value, exc_traceback = sys.exc_info() - print "Timeout error. Function: {0}, error: {1}".format(func.func_code, err) - print repr(traceback.format_exception(exc_type, exc_value,exc_traceback)) - - return try_except - -class Shared(object): - - @staticmethod - @try_except_decor - def option_selection(browser, element_type, element_name, option_text, wait_element_type = '', wait_element_name = ''): - - ret = False - Shared.wait_for_element(browser, element_type, element_name) - if element_type == 'id': - ele = browser.find_element_by_id(element_name) - elif element_type == 'class_name': - ele = browser.find_element_by_class_name(element_name) - options = ele.find_elements_by_tag_name('option') - option_names = [option.text for option in options] - if option_text not in option_names: - return ret - - for option in options: - if option.text.find(option_text) > -1: - option.click() - ret = True - time.sleep(1) - break - - if len(wait_element_type) > 0 and len(wait_element_name) > 0: - Shared.wait_for_element(browser, wait_element_type, wait_element_name) - return ret - - @staticmethod - @try_except_decor - def flash_message(browser): - try: - ele1 = browser.find_element_by_id('flashMessageArea') - except NoSuchElementException: - ele1 = None - if ele1 != None: - ele2 = ele1.find_element_by_class_name('flash_message') - if ele2 != None and ele2.text != None and len(ele2.text) > 0: - return ele2.text - else: - return '' - else: - return '' - - @staticmethod - @try_except_decor - def string_selection(browser, key, value, index = 0): - element = browser.find_elements_by_id(key)[index] - element.clear() - element.send_keys(value) - - @staticmethod - def wait_until_title_text(browser, text, waittime = 30): - wait = WebDriverWait(browser, waittime) - wait.until(lambda browser: browser.title.lower().find(text.lower()) > -1) - - @staticmethod - def wait_until_find_id(browser, element_id, waittime = 10): - wait = WebDriverWait(browser, waittime) - wait.until(lambda browser: browser.find_element_by_id(element_id)) - - @staticmethod - # the name should exist in the newer page, but not in older one - def wait_for_element(browser, element_type, name, waittime = 30): - wait = WebDriverWait(browser, waittime) - if element_type.lower() == 'id': - wait.until(EC.presence_of_element_located((By.ID, name))) - elif element_type.lower() == 'tag_name': - wait.until(EC.presence_of_element_located((By.TAG_NAME, name))) - elif element_type.lower() == 'class_name': - wait.until(EC.presence_of_element_located((By.CLASS_NAME, name))) - elif element_type.lower() == 'xpath': - wait.until(EC.presence_of_element_located((By.XPATH, name))) - elif element_type.lower() == 'link_text': - wait.until(EC.presence_of_element_located((By.LINK_TEXT, name))) - - #feed the string through directly - else: - wait.until(EC.presence_of_element_located(element_type, name)) - - time.sleep(1) - - def playing_around(self): - from threading import Timer - t = Timer(20,self.wait_for_invisible) - t.start() - - @staticmethod - #wait until something disappears - def wait_for_invisible(browser, element_type, name, waittime=30): - wait = WebDriverWait(browser, waittime) - - # the code base uses underscores, but the real string doesn't have em. - final_type = re.sub('_',' ',element_type) - - wait.until(EC.invisibility_of_element_located((final_type, name))) - - #this method isn't as slick as I hoped :( - time.sleep(1) diff --git a/test/selenium/cspages/accounts/accountspage.py b/test/selenium/cspages/accounts/accountspage.py deleted file mode 100644 index 15eccd3ffbc..00000000000 --- a/test/selenium/cspages/accounts/accountspage.py +++ /dev/null @@ -1,175 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from selenium import webdriver -from selenium.common.exceptions import * -from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0 -from selenium.webdriver.common.action_chains import ActionChains as action -from common import Global_Locators -from cspages.cspage import CloudStackPage - -from common.shared import * - -class AccountsPage(CloudStackPage): - - def __init__(self, browser): - self.browser = browser - self.accounts = [] - - @try_except_decor - def get_accounts(self): - rows = self.browser.find_elements_by_xpath("//div[@class='data-table']/table[@class='body']/tbody/tr") - for row in rows: - account = {} - columes = row.find_elements_by_tag_name('td') - account['Name'] = columes[0].get_attribute('title').lower() - account['Role'] = columes[1].get_attribute('title').lower() - account['Domain'] = columes[2].get_attribute('title').lower() - account['State'] = columes[3].get_attribute('title').lower() - self.accounts.append(account) - - @try_except_decor - def account_exists(self, name): - if len(self.accounts) == 0: - self.get_accounts() - account = [acct for acct in self.accounts if acct['Name'] == name.lower()] - if len(account) > 0: - return True - else: - return False - - @try_except_decor - def add_account(self, username = "", password = "", email = "", firstname = "", lastname = "", domain = "", account = "", type = "", timezone = "", network_domain = ""): - # type = role - if len(username) == 0 or len(password) == 0 or len(email) == 0 or len(firstname) == 0 or len(lastname) == 0 or len(domain) == 0 or len(type) == 0: - return; - if type not in ('User', 'Admin'): - print "Account type must be either User or Admin." - return; - if self.account_exists(username): - return - - # click Add Account - ele = self.browser.find_element_by_xpath("//div[@class='toolbar']") - ele1 = ele.find_element_by_xpath("//div[3]/span") - ele1.click() - - Shared.wait_for_element(self.browser, 'id', 'label_username') - ele = self.browser.find_element_by_xpath("(//input[@name='username' and @type='text' and @id='label_username'])") - ele.send_keys(username) - ele = self.browser.find_element_by_xpath("(//input[@name='password' and @type='password' and @id='password'])") - ele.send_keys(password) - ele = self.browser.find_element_by_xpath("(//input[@name='password-confirm' and @type='password' and @id='label_confirm_password'])") - ele.send_keys(password) - ele = self.browser.find_element_by_xpath("(//input[@name='email' and @type='text' and @id='label_email'])") - ele.send_keys(email) - ele = self.browser.find_element_by_xpath("(//input[@name='firstname' and @type='text' and @id='label_first_name'])") - ele.send_keys(firstname) - ele = self.browser.find_element_by_xpath("(//input[@name='lastname' and @type='text' and @id='label_last_name'])") - ele.send_keys(lastname) - Shared.option_selection(self.browser, 'id', 'label_domain', 'ROOT') - if len(account) > 0: - ele = self.browser.find_element_by_xpath("(//input[@name='account' and @type='text' and @id='label_account'])") - ele.send_keys(account) - Shared.option_selection(self.browser, 'id', 'label_type', type) - Shared.option_selection(self.browser, 'id', 'label_timezone', timezone) - if len(network_domain) > 0: - ele = self.browser.find_element_by_xpath("(//input[@name='networkdomain' and @type='text' and @id='label_network_domain'])") - ele.send_keys(network_domain) - self.button_add() - - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def select_account(self, username = "", domain = "", type = ""): - if len(username) == 0 or len(domain) == 0 or len(type) == 0: - return False; - if self.account_exists(username) == False: - print "The account does not exist" - return False - - # select the account - ele = self.browser.find_element_by_xpath("//div[@class='data-table']/div[@class='fixed-header']/table") - ele1 = ele.find_element_by_xpath("//tbody") - ele2 = ele1.find_elements_by_tag_name('tr') - for e in ele2: - ele3 = e.find_elements_by_tag_name('td') - # move mouse to quickview - if len(ele3) > 4 and \ - ele3[0].text == username and \ - ele3[1].text == type and \ - ele3[2].text == domain and \ - ele3[3].text == 'enabled': - ele3[4].find_element_by_tag_name('span').click() - Shared.wait_for_element(self.browser, 'class_name', 'details') - # select account - ele = self.browser.find_element_by_xpath("//div[@id='details-tab-details']/div[@class='details']/div/table/tbody/tr/td[@class='view-all']") - ele1 = ele.find_element_by_tag_name('a').find_element_by_tag_name('span').click() - break - - Shared.wait_for_element(self.browser, 'class_name', 'view-all') - - @try_except_decor - def delete_account(self, username = "", domain = "", type = ""): - if len(username) == 0 or len(domain) == 0 or len(type) == 0: - return False; - if self.account_exists(username) == False: - print "The account does not exist" - return False - - # find the account - ele = self.browser.find_element_by_xpath("//div[@class='data-table']/div[@class='fixed-header']/table") - ele1 = ele.find_element_by_xpath("//tbody") - ele2 = ele1.find_elements_by_tag_name('tr') - for e in ele2: - ele3 = e.find_elements_by_tag_name('td') - # move mouse to quickview - if len(ele3) > 4 and \ - ele3[0].text == username and \ - ele3[1].text == type and \ - ele3[2].text == domain and \ - ele3[3].text == 'enabled': - ele3[4].find_element_by_tag_name('span').click() - Shared.wait_for_element(self.browser, 'class_name', 'details') - # delete account - ele = self.browser.find_element_by_xpath("//div[@id='details-tab-details']/div[@class='details']/div/table/tbody/tr/td/div[@class='buttons']") - ele1 = ele.find_element_by_xpath("//div[@class='action remove single text' and @title='Delete account']/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'ui-dialog-buttonset') - self.button_yes() - break - - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_cancel(self): - ele = self.browser.find_element_by_xpath("/html/body/div[4]/div[2]/div/button[1]/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_add(self): - ele = self.browser.find_element_by_xpath("/html/body/div[4]/div[2]/div/button[2]/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_no(self): - ele = self.browser.find_element_by_xpath("/html/body/div[4]/div[10]/div/button[1]/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_yes(self): - ele = self.browser.find_element_by_xpath("/html/body/div[4]/div[10]/div/button[2]/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') diff --git a/test/selenium/cspages/accounts/userspage.py b/test/selenium/cspages/accounts/userspage.py deleted file mode 100644 index 25375f5d9ad..00000000000 --- a/test/selenium/cspages/accounts/userspage.py +++ /dev/null @@ -1,146 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from selenium import webdriver -from selenium.common.exceptions import * -from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0 -from selenium.webdriver.common.action_chains import ActionChains as action -from common import Global_Locators -from cspages.cspage import CloudStackPage - -from common.shared import * - -class UsersPage(CloudStackPage): - - def __init__(self, browser): - self.browser = browser - self.users = [] - - @try_except_decor - def get_users(self): - ele = self.browser.find_element_by_xpath("//div[@class='container cloudStack-widget cloudBrowser']") - rows = ele.find_elements_by_xpath("//div[@class='panel']/div[2]/div[@class='view list-view']/div[@class='data-table']/table[@class='body']/tbody/tr") - for row in rows: - user = {} - columes = row.find_elements_by_tag_name('td') - user['username'] = columes[0].get_attribute('title').lower() - user['firstname'] = columes[1].get_attribute('title').lower() - user['lastname'] = columes[2].get_attribute('title').lower() - self.users.append(user) - - @try_except_decor - def user_exists(self, username): - if len(self.users) == 0: - self.get_users() - users = [u for u in self.users if u['username'] == username.lower()] - if len(users) > 0: - return True - else: - return False - - @try_except_decor - def add_user(self, username = "", password = "", email = "", firstname = "", lastname = "", timezone = ""): - if len(username) == 0 or len(password) == 0 or len(email) == 0 or len(firstname) == 0 or len(lastname) == 0: - return; - if self.user_exists(username): - return - - # click Add User - ele = self.browser.find_element_by_xpath("//div[@class='container cloudStack-widget cloudBrowser']") - ele1 = ele.find_element_by_xpath("//div[@class='panel']/div[2]/div[@class='view list-view']/div[@class='toolbar']/div[@class='button action add reduced-hide']/span") - ele1.click() - - Shared.wait_for_element(self.browser, 'id', 'label_username') - ele = self.browser.find_element_by_xpath("(//input[@name='username' and @type='text' and @id='label_username'])") - ele.send_keys(username) - ele = self.browser.find_element_by_xpath("(//input[@name='password' and @type='password' and @id='password'])") - ele.send_keys(password) - ele = self.browser.find_element_by_xpath("(//input[@name='password-confirm' and @type='password' and @id='label_confirm_password'])") - ele.send_keys(password) - ele = self.browser.find_element_by_xpath("(//input[@name='email' and @type='text' and @id='label_email'])") - ele.send_keys(email) - ele = self.browser.find_element_by_xpath("(//input[@name='firstname' and @type='text' and @id='label_first_name'])") - ele.send_keys(firstname) - ele = self.browser.find_element_by_xpath("(//input[@name='lastname' and @type='text' and @id='label_last_name'])") - ele.send_keys(lastname) - Shared.option_selection(self.browser, 'id', 'label_timezone', timezone) - self.button_ok() - - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def delete_user(self, username = "", firstname = "", lastname = ""): - if len(username) == 0 or len(firstname) == 0 or len(lastname) == 0: - return False; - if self.user_exists(username) == False: - print "The user does not exist" - return False - - # find the user - ele = self.browser.find_element_by_xpath("//div[@class='container cloudStack-widget cloudBrowser']") - ele1 = ele.find_element_by_xpath("//div[@class='panel']/div[2]/div[@class='view list-view']/div[@class='data-table']/table[@class='body']/tbody") - ele2 = ele1.find_elements_by_tag_name('tr') - for e in ele2: - ele3 = e.find_elements_by_tag_name('td') - # move mouse to quickview - if len(ele3) > 3 and \ - ele3[0].text == username and \ - ele3[1].text == firstname and \ - ele3[2].text == lastname: - ele3[3].find_element_by_tag_name('span').click() - Shared.wait_for_element(self.browser, 'class_name', 'details') - # delete user - ele = self.browser.find_element_by_xpath("//div[@id='details-tab-details']/div[@class='details']/div/table/tbody/tr/td/div[@class='buttons']") - ele1 = ele.find_element_by_xpath("//div[@class='action remove single text' and @title='Delete User']/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'ui-dialog-buttonset') - self.button_yes() - break - - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_yes(self): - eles = self.browser.find_elements_by_xpath("//div[@class='ui-dialog-buttonset']/button[@type='button' and @role='button']") - for e in eles: - ele = e.find_element_by_class_name('ui-button-text') - if e.text == 'Yes': - e.click() - break - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_no(self): - ele = self.browser.find_element_by_xpath("/html/body/div[4]/div[10]/div/button[1]/span").click() - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_ok(self): - eles = self.browser.find_elements_by_xpath("//button[@type='button' and @role='button']") - for e in eles: - if e.text == 'OK': - e.click() - break - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') - - @try_except_decor - def button_cancel(self): - eles = self.browser.find_elements_by_xpath("//button[@type='button' and @role='button']") - for e in eles: - if e.text == 'Cancel': - e.click() - break - Shared.wait_for_element(self.browser, 'class_name', 'fixed-header') diff --git a/test/selenium/cspages/dashboard/dashboardpage.py b/test/selenium/cspages/dashboard/dashboardpage.py deleted file mode 100644 index 12a38a1978b..00000000000 --- a/test/selenium/cspages/dashboard/dashboardpage.py +++ /dev/null @@ -1,73 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from selenium import webdriver -from selenium.common.exceptions import * -from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0 -from selenium.webdriver.common.action_chains import ActionChains as action -from common import Global_Locators -from cspages.cspage import CloudStackPage - -from common.shared import * - -class DashboardPage(CloudStackPage): - - def __init__(self, browser): - self.browser = browser - self.active_item = "" - self.items = [] - - @try_except_decor - def get_active_item(self): - self.active_item = "" - lis = self.browser.find_elements_by_xpath("//*[@id='navigation']/ul/li") - for li in lis: - if li.get_attribute('class').find('active') > 0: - self.active_item = li.get_attribute('class')[:(li.get_attribute('class').index(' active'))] - return self.active_item - - @try_except_decor - def get_items(self): - lis = self.browser.find_elements_by_xpath("//*[@id='navigation']/ul/li") - for li in lis: - item = li.get_attribute('class')[len('navigation-item '):] - if item.find('active') > 0: - item = item[:(item.index(' active'))] - if item.find('first') > 0: - item = item[:(item.index(' first'))] - if item.find('last') > 0: - item = item[:(item.index(' last'))] - self.items.append(item.lower()) - return self.items -# import pdb -# pdb.set_trace() - - @try_except_decor - def navigate_to(self, item_name): - if len(self.items) == 0: - self.get_items() - if item_name is None or len(item_name) == 0 or \ - item_name.lower() not in self.items or \ - (len(self.active_item) > 0 and self.active_item.lower().find(item_name.lower()) > 0): - return - - lis = self.browser.find_elements_by_xpath("//*[@id='navigation']/ul/li") - for li in lis: - if li.get_attribute('class').lower().find(item_name.lower()) > 0: - li.click() - time.sleep(3) - return diff --git a/test/selenium/cspages/login/__init__.py b/test/selenium/cspages/login/__init__.py deleted file mode 100644 index 13a83393a91..00000000000 --- a/test/selenium/cspages/login/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/test/selenium/cspages/login/loginpage.py b/test/selenium/cspages/login/loginpage.py deleted file mode 100644 index f4b0e8adba2..00000000000 --- a/test/selenium/cspages/login/loginpage.py +++ /dev/null @@ -1,103 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from selenium import webdriver -from selenium.common.exceptions import * -from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0 -from selenium.webdriver.common.action_chains import ActionChains as action -from common import Global_Locators -from cspages.cspage import CloudStackPage - -from common.shared import * - -import pdb - -class LoginPage(CloudStackPage): - def __init__(self, browser): - self.browser = browser - self.username = "" - self.password = "" - self.language = "" - - @try_except_decor - def set_username(self, username): - self.username = username - usernameElement = self.browser.find_element_by_css_selector(Global_Locators.login_username_css) - usernameElement.send_keys(self.username) - - @try_except_decor - def set_password(self, password): - self.password = password - passwordElement = self.browser.find_element_by_css_selector(Global_Locators.login_password_css) - passwordElement.send_keys(self.password) - self.pwelement = passwordElement - - @try_except_decor - def set_language(self, language): - self.language = language - options = self.browser.find_elements_by_xpath('/html/body/div[3]/form/div[2]/div[4]/select/option') - for option in options: - if len(option.get_attribute('text')) > 0 and option.get_attribute('text').lower() == language.lower(): - option.click() - break - - @try_except_decor - def login(self, expect_fail = False): - if self.username == "" or self.password == "": - print "Must set email and password before logging in" - return - loginElement = self.browser.find_element_by_css_selector(Global_Locators.login_submit_css) - loginElement.click() - - time.sleep(3) - try: - # in case we have that "Hello and Welcome to CloudStack" page - ele = None - ele = self.browser.find_element_by_xpath("//input[@type='submit' and @class='button goTo advanced-installation' and @value='I have used CloudStack before, skip this guide']") - if ele is not None: - ele.click() - time.sleep(2) - except NoSuchElementException as err: - pass - - @try_except_decor - def logout(self, directly_logout = False): - - Shared.wait_for_element(self.browser, 'id', 'user') - - # must click this icon options first - if directly_logout == False: - try: - ele = self.browser.find_element_by_xpath("//div[@id='user-options' and @style='display: block;']") - if ele is None: - ele1 = self.browser.find_element_by_xpath("//div[@id='user' and @class='button']/div[@class='icon options']/div[@class='icon arrow']").click() - except NoSuchElementException as err: - ele1 = self.browser.find_element_by_xpath("//div[@id='user' and @class='button']/div[@class='icon options']/div[@class='icon arrow']").click() - time.sleep(1) - - ele2 = self.browser.find_element_by_xpath("//div[@id='user' and @class='button']/div[@id='user-options']/a[1]").click() - - Shared.wait_for_element(self.browser, 'class_name', 'login') - - @try_except_decor - def get_error_msg(self, loginpage_url): - if loginpage_url is not None and len(loginpage_url) > 0 and \ - (self.browser.current_url.find(loginpage_url) > -1 or loginpage_url.find(self.browser.current_url) > -1): - ele = self.browser.find_element_by_id('std-err') - return ele.text - else: - return "" diff --git a/test/selenium/cstests/__init__.py b/test/selenium/cstests/__init__.py deleted file mode 100644 index 13a83393a91..00000000000 --- a/test/selenium/cstests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/test/selenium/cstests/regressiontests/__init__.py b/test/selenium/cstests/regressiontests/__init__.py deleted file mode 100644 index 13a83393a91..00000000000 --- a/test/selenium/cstests/regressiontests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/test/selenium/cstests/smoketests/__init__.py b/test/selenium/cstests/smoketests/__init__.py deleted file mode 100644 index 13a83393a91..00000000000 --- a/test/selenium/cstests/smoketests/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. diff --git a/test/selenium/cstests/smoketests/adduser_test.py b/test/selenium/cstests/smoketests/adduser_test.py deleted file mode 100644 index b19a0601158..00000000000 --- a/test/selenium/cstests/smoketests/adduser_test.py +++ /dev/null @@ -1,103 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg -import cspages.login.loginpage as loginpage -import cspages.dashboard.dashboardpage as dashboardpage -import cspages.accounts.accountspage as accountspage -import cspages.accounts.userspage as userspage - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSAddUser(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - - shared.Shared.wait_for_element(self.browser.browser, 'id', 'navigation') - - time.sleep(3) - - self.dashboardpage = dashboardpage.DashboardPage(self.browser.get_browser()) - - # navigate to Accounts page - self.dashboardpage.navigate_to('accounts') - - # make sure we are on Accounts page - activeitem = self.dashboardpage.get_active_item() - if activeitem.find('accounts') < 0: - self.assertRaises(ValueError, self.dashboardpage.get_active_item(), activeitem) - - # now we are at Accounts page - self.accountspage = accountspage.AccountsPage(self.browser.get_browser()) - self.accountspage.select_account(username = smokecfg['account']['username'], - domain = smokecfg['account']['domain'], - type = smokecfg['account']['type'], - ) - - # now we are at users page - self.userspage = userspage.UsersPage(self.browser.get_browser()) - self.userspage.add_user(username = smokecfg['new user']['username'], - password = smokecfg['new user']['password'], - email = smokecfg['new user']['email'], - firstname = smokecfg['new user']['firstname'], - lastname = smokecfg['new user']['lastname'], - timezone = smokecfg['new user']['timezone'], - ) - - self.loginpage.logout() - - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login') - - def xtest_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/adduseraccount_test.py b/test/selenium/cstests/smoketests/adduseraccount_test.py deleted file mode 100644 index e687c6b53a5..00000000000 --- a/test/selenium/cstests/smoketests/adduseraccount_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg -import cspages.login.loginpage as loginpage -import cspages.dashboard.dashboardpage as dashboardpage -import cspages.accounts.accountspage as accountspage - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSAddUserAccount(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - - shared.Shared.wait_for_element(self.browser.browser, 'id', 'navigation') - - time.sleep(3) - - self.dashboardpage = dashboardpage.DashboardPage(self.browser.get_browser()) - - # navigate to Accounts page - self.dashboardpage.navigate_to('accounts') - - # make sure we are on Accounts page - activeitem = self.dashboardpage.get_active_item() - if activeitem.find('accounts') < 0: - self.assertRaises(ValueError, self.dashboardpage.get_active_item(), activeitem) - - # now we are at Accounts page - self.accountspage = accountspage.AccountsPage(self.browser.get_browser()) - self.accountspage.add_account(username = smokecfg['new user account']['username'], - password = smokecfg['new user account']['password'], - email = smokecfg['new user account']['email'], - firstname = smokecfg['new user account']['firstname'], - lastname = smokecfg['new user account']['lastname'], - domain = smokecfg['new user account']['domain'], - type = smokecfg['new user account']['type'], - timezone = smokecfg['new user account']['timezone'], - ) - self.loginpage.logout() - - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login') - - def xtest_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/deleteuser_test.py b/test/selenium/cstests/smoketests/deleteuser_test.py deleted file mode 100644 index 8d3d28d68ee..00000000000 --- a/test/selenium/cstests/smoketests/deleteuser_test.py +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg -import cspages.login.loginpage as loginpage -import cspages.dashboard.dashboardpage as dashboardpage -import cspages.accounts.accountspage as accountspage -import cspages.accounts.userspage as userspage - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSDeleteUser(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - - shared.Shared.wait_for_element(self.browser.browser, 'id', 'navigation') - - time.sleep(3) - - self.dashboardpage = dashboardpage.DashboardPage(self.browser.get_browser()) - - # navigate to Accounts page - self.dashboardpage.navigate_to('accounts') - - # make sure we are on Accounts page - activeitem = self.dashboardpage.get_active_item() - if activeitem.find('accounts') < 0: - self.assertRaises(ValueError, self.dashboardpage.get_active_item(), activeitem) - - # now we are at Accounts page - self.accountspage = accountspage.AccountsPage(self.browser.get_browser()) - self.accountspage.select_account(username = smokecfg['account']['username'], - domain = smokecfg['account']['domain'], - type = smokecfg['account']['type'], - ) - - # now we are at users page - self.userspage = userspage.UsersPage(self.browser.get_browser()) - self.userspage.delete_user(username = smokecfg['new user']['username'], - firstname = smokecfg['new user']['firstname'], - lastname = smokecfg['new user']['lastname'], - ) - - self.loginpage.logout() - - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login') - - def xtest_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/deleteuseraccount_test.py b/test/selenium/cstests/smoketests/deleteuseraccount_test.py deleted file mode 100644 index 179646adeea..00000000000 --- a/test/selenium/cstests/smoketests/deleteuseraccount_test.py +++ /dev/null @@ -1,91 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg -import cspages.login.loginpage as loginpage -import cspages.dashboard.dashboardpage as dashboardpage -import cspages.accounts.accountspage as accountspage - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSDeleteAccount(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - - shared.Shared.wait_for_element(self.browser.browser, 'id', 'navigation') - - time.sleep(3) - - self.dashboardpage = dashboardpage.DashboardPage(self.browser.get_browser()) - - # navigate to Accounts page - self.dashboardpage.navigate_to('accounts') - - # make sure we are on Accounts page - activeitem = self.dashboardpage.get_active_item() - if activeitem.find('accounts') < 0: - self.assertRaises(ValueError, self.dashboardpage.get_active_item(), activeitem) - - # now we are at Accounts page - self.accountspage = accountspage.AccountsPage(self.browser.get_browser()) - self.accountspage.delete_account(username = smokecfg['new user account']['username'], - domain = smokecfg['new user account']['domain'], - type = smokecfg['new user account']['type'], - ) - self.loginpage.logout() - - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login') - - def xtest_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/global_settings_test.py b/test/selenium/cstests/smoketests/global_settings_test.py deleted file mode 100644 index 2fc0ff194d0..00000000000 --- a/test/selenium/cstests/smoketests/global_settings_test.py +++ /dev/null @@ -1,69 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg -import cspages.login.loginpage as loginpage -import cspages.dashboard.dashboardpage as dashboardpage - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSGlobalSettings(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - shared.Shared.wait_for_element(self.browser.browser, 'id', 'navigation') - - time.sleep(3) - - self.dashboardpage = dashboardpage.DashboardPage(self.browser.get_browser()) - active_item = self.dashboardpage.get_active_item() - print "active item: ", active_item - - self.loginpage.logout() - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login') - - def xtest_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/login_logout_as_JohnD_test.py b/test/selenium/cstests/smoketests/login_logout_as_JohnD_test.py deleted file mode 100644 index a01f2f8fbcb..00000000000 --- a/test/selenium/cstests/smoketests/login_logout_as_JohnD_test.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import cspages.login.loginpage as loginpage -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSLoginLogout(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['new user account']['username']) - self.loginpage.set_password(smokecfg['new user account']['password']) - self.loginpage.login() - - time.sleep(5) - - self.loginpage.logout(directly_logout = True) - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login', waittime = 300) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/login_logout_test.py b/test/selenium/cstests/smoketests/login_logout_test.py deleted file mode 100644 index c94e4e32294..00000000000 --- a/test/selenium/cstests/smoketests/login_logout_test.py +++ /dev/null @@ -1,190 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import cspages.login.loginpage as loginpage -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSLoginLogout(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - shared.Shared.wait_for_element(self.browser.get_browser(), 'id', 'navigation', waittime = 300) - - time.sleep(5) - - self.loginpage.logout(directly_logout = True) - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login', waittime = 300) - - def test_failure_1(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['badusername']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - - def test_failure_2(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['badpassword']) - self.loginpage.login(expect_fail = True) - - def test_failure_3(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['badusername']) - self.loginpage.set_password(smokecfg['badpassword']) - self.loginpage.login(expect_fail = True) - - def test_failure_4(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['sqlinjection_1']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - - def test_failure_5(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['sqlinjection_2']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - - def test_failure_6(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['sqlinjection_3']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - - def test_failure_7(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['sqlinjection_4']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - - def test_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/navigation_test.py b/test/selenium/cstests/smoketests/navigation_test.py deleted file mode 100644 index 01ea449505e..00000000000 --- a/test/selenium/cstests/smoketests/navigation_test.py +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import sys, os, time -import json - -sys.path.append('./') - -import browser.firefox as firefox -import common.shared as shared -from cstests.smoketests.smokecfg import smokecfg -import cspages.login.loginpage as loginpage -import cspages.dashboard.dashboardpage as dashboardpage - -# from cstests.smoketests import smokecfg as smokecfg - -class TestCSnavigation(unittest.TestCase): - def setUp(self): - # Create a new instance of the Firefox browser - self.browser = firefox.Firefox('firefox') - - def tearDown(self): - self.browser.quit_browser() - - def test_success(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - - # wait for at most 5 minutes, in case we have an anoyingly slow server - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'select-language', waittime = 300) - - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - - shared.Shared.wait_for_element(self.browser.get_browser(), 'class_name', 'fields', waittime = 300) - - self.loginpage.set_username(smokecfg['username']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login() - - shared.Shared.wait_for_element(self.browser.browser, 'id', 'navigation') - - time.sleep(3) - - self.dashboardpage = dashboardpage.DashboardPage(self.browser.get_browser()) - - items = self.dashboardpage.get_items() - [self.dashboardpage.navigate_to(item) for item in items] - - self.loginpage.logout() - - shared.Shared.wait_for_element(self.browser.browser, 'class_name', 'login') - - def xtest_failure_8(self): - self.browser.set_url(smokecfg['cssite']) - self.loginpage = loginpage.LoginPage(self.browser.get_browser()) - # language selection must be done before username and password - self.loginpage.set_language(smokecfg['language']) - self.loginpage.set_username(smokecfg['sqlinjection_5']) - self.loginpage.set_password(smokecfg['password']) - self.loginpage.login(expect_fail = True) - -if __name__ == '__main__': - unittest.main() diff --git a/test/selenium/cstests/smoketests/smokecfg.py b/test/selenium/cstests/smoketests/smokecfg.py deleted file mode 100644 index b6c09a660d9..00000000000 --- a/test/selenium/cstests/smoketests/smokecfg.py +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -#!/usr/bin/python -# coding: latin-1 - -smokecfg = { - 'browser': 'Firefox', -# 'window position': '10, 10', # upper left coordinates -# 'window size': '2000, 1500', - 'cssite': 'http://127.0.0.1:8080/client/', -# 'cssite': 'http://192.168.1.31:8080/client/', - 'username': 'admin', - 'password': 'password', - 'badusername': 'badname', - 'badpassword': 'badpassword', - 'sqlinjection_1': '\' or 1=1 --\'', - 'sqlinjection_2': '\' union select 1, badusername, badpassword, 1--\'', - 'sqlinjection_3': '\' union select @@version,1,1,1--\'', - 'sqlinjection_4': '\'; drop table user--\'', - 'sqlinjection_5': '\'OR\' \'=\'', - 'language': 'English', - - # add a new user account - 'new user account':{'username': 'JohnD', - 'password': 'password', - 'email': 'johndoe@aol.com', - 'firstname': 'John', - 'lastname': 'Doe', - 'domain': 'ROOT', - 'type': 'User', # either 'User' or 'Admin' - 'timezone': 'US/Eastern [Eastern Standard Time]', - }, - # add a new user under JohnD - 'account': {'username': 'JohnD', - 'domain': 'ROOT', - 'type': 'User', - }, - # add a new user - 'new user': {'username': 'JaneD', - 'password': 'password', - 'email': 'janedoe@aol.com', - 'firstname': 'Jane', - 'lastname': 'Doe', - 'timezone': 'US/Eastern [Eastern Standard Time]', - }, - - } diff --git a/test/selenium/lib/Global_Locators.py b/test/selenium/lib/Global_Locators.py deleted file mode 100644 index 111e1549353..00000000000 --- a/test/selenium/lib/Global_Locators.py +++ /dev/null @@ -1,222 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' -Variable Names are as follows -Logical Page Descriptor_____What Element Represents and/or where it is_____LocatorType - - -For Example :: - -instances_xpath = "//div[@id='navigation']/ul/li[2]/span[2]" - -Means this is:: xpath link for Instances which is present on Dashboard. -Any test cases that requires to go into Instances from Dashboard can use this variable now. - -This may not be intuitive as you go deep into the tree. - - - -for example - -stopinstanceforce_id - -The best way to know what this represents is to track by variable name -Under Instances / any instance is click on any instance (applies to any instance) / stop instance has a force stop check box when you click. -This link represents that. - - -Steps below do not have global locators. - -PF rule steps including and after filling port numbers. (Refer to vmLifeAndNetwork.py / def test_PF) -FW rule steps including and after filling port numbers. (Refer to vmLifeAndNetwork.py / def test_PF) -ADD Disk Offering page has Names, description, storage type etc etc -ADD Compute Offering page has Names, description, CPU Cores, CPU clocks type etc etc - -Create Acc, Delete Acc, Login and Logout are for test flow and are not test cases. They do not have global Locators. - -Such and many more data entry points that appear only once and hence we do not need glonal names for them. They are hard coded as and when needed in the scripts. - - -''' - -################################################################################################################################################################################################ - -## Links on the Main UI page (Dash board). Listed in the order they appear on screen -dashboard_xpath = "//div[@id='navigation']/ul/li" -instances_xpath = "//div[@id='navigation']/ul/li[2]/span[2]" # Link for Instance and following as self explanatory -storage_xpath = "//div[@id='navigation']/ul/li[3]/span[2]" -network_xpath = "//div[@id='navigation']/ul/li[4]/span[2]" -templates_xpath = "//div[@id='navigation']/ul/li[5]/span[2]" -events_xpath = "//div[@id='navigation']/ul/li[6]/span[2]" -projects_xpath = "//div[@id='navigation']/ul/li[7]/span[2]" -accounts_xpath = "//div[@id='navigation']/ul/li[8]/span[2]" -domains_xpath = "//div[@id='navigation']/ul/li[9]/span[2]" -infrastructure_xpath = "//div[@id='navigation']/ul/li[10]/span[2]" -globalSettings_xpath = "//div[@id='navigation']/ul/li[11]/span[2]" -serviceOfferings_xpath = "//div[@id='navigation']/ul/li[12]/span[2]" - -################################################################################################################################################################################################ - -## Instances Page -## Instances Main page - - -# Add Instance Button on top right corner of Instances page -add_instance_xpath = "//div[2]/div/div[2]/div/div[2]/span" - -# Add Instance Wizard next button -add_instance_next_xpath = "//div[4]/div[2]/div[3]/div[3]/span" - -# Table that lists all VM's under Instances page; General usage is to traverse through this table and search for the VM we are interested in. -instances_table_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div[2]/table/tbody/tr/td/span" - - -# Click any instance and following are available - -# Click ok on confirmation pop-up box for most actions listed below -actionconfirm_xpath = ("//button[@type='button']") - -# status of VM running. Click on VM > 3rd row in table -state_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div[2]/div[2]/div/div/div[2]/div/table/tbody/tr[3]/td[2]/span" - -# Stop instance icon -stopinstance_css = "a[alt=\"Stop Instance\"] > span.icon" - -# stop instance forcefully check box available after stop instance is executed in separate pop up -stopinstanceforce_id = ("force_stop") - -# start instance icon -startinstance_css = "a[alt=\"Start Instance\"] > span.icon" - -yesconfirmation_xapth = "(//button[@type='button'])[2]" - - -# Destroy instance icon -destroyinstance_css = "a[alt=\"Destroy Instance\"] > span.icon" - -#Restore Instance icon -restoreinstance_css = "a[alt=\"Restore Instance\"] > span.icon" - -# Reboot instance -rebootinstance_css = "a[alt=\"Reboot Instance\"] > span.icon" - -################################################################################################################################################################################################ - - -## Network Page - -# Table that lists all Networks under Network page; General usage is to traverse through this table and search for the network we are interested in. -network_networktable_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div[3]/div[2]/div/div[2]/table/tbody/tr/td/span" - -# View IP addresses button on each network page -viewIp_css="div.view-all > a > span" - -# Acquire a new ip -acquireIP_xpath="//div[2]/div/div/div[2]/span" -# List of IP's within a netork table -network_iptables_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div[3]/div[2]/div/div[2]/table/tbody/tr/td/span" -# Configuration tab for each IP -ipConfiguration_text="Configuration" -# PF under configuration for each IP -ip_PF = "li.portForwarding > div.view-details" - - -################################################################################################################################################################################################ - - -## Servivce Offering Page - -# Selects Compute offering from drop down menu -Offering_compute_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[1]" - -# Selects System offering from drop down menu -Offering_system_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[2]" - -# Selects Disk offering from drop down menu -Offering_disk_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[3]" - -# Selects Network offering from drop down menu -Offering_network_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[4]" - -# Add Offering -Offering_add_xpath ="//div[3]/span" - -# Points to tbale that lists Offerings -Offering_table_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div[2]/table/tbody/tr/td/span" - -# Edit Button -Offering_edit_css = "a[alt=\"Edit\"] > span.icon" - -# Edit name box -Offering_editname_name = "name" - -# Edit description box -Offering_editdescription_name = "displaytext" - -# Edit finished click ok -Offering_editdone_css="div.button.done" - -# delete offering button for Disk only -Offering_delete_css = "a[alt=\"Delete Disk Offering\"] > span.icon" - -# delete offering button for Compute only -Offering_deletecompute_css = "a[alt=\"Delete Service Offering\"] > span.icon" - - - - -################################################################################################################################################################################################ - - -#### Templates Page - -# Selects Templates from drop down -template_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[1]" - -# Selects ISO from drop down -iso_xpath = "/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div/div/div/select/option[2]" - -# Add Template -AddTemplate_xpath = "//div[3]/span" - -# Points to table where all templates are -template_table_xpath ="/html/body/div/div/div[2]/div[2]/div[2]/div/div[2]/div[2]/table/tbody/tr/td/span" - -# Edit Template Button -template_edit_css = "a[alt=\"Edit\"] > span.icon" - -# Edit finished click OK -template_editdone_css = "div.button.done" - -# Delete Template button -template_delete_css = "a[alt=\"Delete Template\"] > span.icon" - - -################################################################################################################################################################################################ - - -## Login Page - -# Username box -login_username_css = "body.login > div.login > form > div.fields > div.field.username > input[name=\"username\"]" # Login>Username TextBox - -# Password Box -login_password_css = "body.login > div.login > form > div.fields > div.field.password > input[name=\"password\"]" # LoginPassword TextBox - -# Click ok to login -login_submit_css = "body.login > div.login > form > div.fields > input[type=\"submit\"]" # Login>Login Button (Submit button) diff --git a/test/selenium/smoke/Login_and_Accounts.py b/test/selenium/smoke/Login_and_Accounts.py deleted file mode 100644 index 2b3aee420e8..00000000000 --- a/test/selenium/smoke/Login_and_Accounts.py +++ /dev/null @@ -1,254 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys, os -sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/'+'../lib')) - - -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import Select -from selenium.common.exceptions import NoSuchElementException -import unittest, time -import Global_Locators -import initialize - - - -class login(unittest.TestCase): - - - def setUp(self): - - MS_URL = initialize.getMSip() - self.driver = initialize.getOrCreateWebdriver() - self.base_url = "http://"+ MS_URL +":8080/" # Your management Server IP goes here - self.verificationErrors = [] - - - def test_login(self): - - # Here we will clear the test box for Username and Password and fill them with actual login data. - # After that we will click Login (Submit button) - driver = self.driver - driver.maximize_window() - driver.get(self.base_url + "client/") - driver.find_element_by_css_selector(Global_Locators.login_username_css).clear() - driver.find_element_by_css_selector(Global_Locators.login_username_css).send_keys("admin") - driver.find_element_by_css_selector(Global_Locators.login_password_css).clear() - driver.find_element_by_css_selector(Global_Locators.login_password_css).send_keys("password") - driver.find_element_by_css_selector(Global_Locators.login_submit_css).click() - time.sleep(5) - - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - -################################################################################################################################################ - - - -class logout(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.driver.implicitly_wait(100) - self.verificationErrors = [] - - - - def test_logout(self): - - # Here we will clear the test box for Username and Password and fill them with actual login data. - # After that we will click Login (Submit button) - driver = self.driver - driver.find_element_by_xpath("//div[@id='navigation']/ul/li").click() - driver.find_element_by_css_selector("div.icon.options").click() - driver.find_element_by_link_text("Logout").click() - - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - -################################################################################################################################################ - - - -class login_test(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_logintest(self): - - # Here we will clear the test box for Username and Password and fill them with actual login data. - # After that we will click Login (Submit button) - driver = self.driver - driver.find_element_by_css_selector(Global_Locators.login_username_css).clear() - driver.find_element_by_css_selector(Global_Locators.login_username_css).send_keys("test") - driver.find_element_by_css_selector(Global_Locators.login_password_css).clear() - driver.find_element_by_css_selector(Global_Locators.login_password_css).send_keys("password") - driver.find_element_by_css_selector(Global_Locators.login_submit_css).click() - time.sleep(5) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - -################################################################################################################################################ - - -class createAcc(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_createacc(self): - - driver = self.driver - self.driver.implicitly_wait(100) - driver.find_element_by_xpath("//div[@id='navigation']/ul/li[8]/span[2]").click() - driver.find_element_by_xpath("//div[3]/span").click() - driver.find_element_by_id("label_username").clear() - driver.find_element_by_id("label_username").send_keys("test") - driver.find_element_by_id("password").clear() - driver.find_element_by_id("password").send_keys("password") - driver.find_element_by_id("label_confirm_password").clear() - driver.find_element_by_id("label_confirm_password").send_keys("password") - driver.find_element_by_id("label_email").clear() - driver.find_element_by_id("label_email").send_keys("test@citrix.com") - driver.find_element_by_id("label_first_name").clear() - driver.find_element_by_id("label_first_name").send_keys("test") - driver.find_element_by_id("label_last_name").clear() - driver.find_element_by_id("label_last_name").send_keys("test") - driver.find_element_by_id("label_domain").click() - Select(driver.find_element_by_id("label_type")).select_by_visible_text("Admin") - Select(driver.find_element_by_id("label_timezone")).select_by_visible_text("[UTC-08:00] Pacific Standard Time") - driver.find_element_by_xpath("//button[@type='button']").click() - - # Go to Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(30) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - -################################################################################################################################################ - - -class tearAcc(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_tearacc(self): - - driver = self.driver - driver.find_element_by_css_selector("li.navigation-item.accounts").click() - driver.find_element_by_css_selector("tr.odd > td.name.first").click() - driver.find_element_by_css_selector("a[alt=\"Delete account\"] > span.icon").click() - driver.find_element_by_xpath("(//button[@type='button'])[2]").click() - - # Go to Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(30) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - def tearDown(self): - - self.driver.quit() - self.assertEqual([], self.verificationErrors) - - - -################################################################################################################################################ diff --git a/test/selenium/smoke/Service_Offering.py b/test/selenium/smoke/Service_Offering.py deleted file mode 100644 index 5c8a4475ba5..00000000000 --- a/test/selenium/smoke/Service_Offering.py +++ /dev/null @@ -1,426 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys, os -sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/'+'../lib')) - -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import Select -from selenium.common.exceptions import NoSuchElementException -import unittest, time -import initialize -import Global_Locators - - - - -class Disk_offering_Add(unittest.TestCase): - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_diskadd(self): - - driver = self.driver - self.driver.implicitly_wait(200) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Service Offerings - driver.find_element_by_xpath(Global_Locators.serviceOfferings_xpath).click() - - #Select Disk offering - driver.find_element_by_xpath(Global_Locators.Offering_disk_xpath).click() - - # Add offering - driver.find_element_by_xpath(Global_Locators.Offering_add_xpath).click() - - # Following have names.. so they do not have their global entries. - driver.find_element_by_name("name").clear() - driver.find_element_by_name("name").send_keys("Test Disk Name") - driver.find_element_by_name("description").clear() - driver.find_element_by_name("description").send_keys("Test Disk Description") - driver.find_element_by_name("disksize").clear() - driver.find_element_by_name("disksize").send_keys("1") - driver.find_element_by_xpath("//button[@type='button']").click() - time.sleep(20) - - ##Verification will be if this offering shows up into table and we can actually edit it. - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - def tearDown(self): - self.assertEqual([], self.verificationErrors) - - - - - -class Disk_offering_Edit(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_diskedit(self): - - driver = self.driver - self.driver.implicitly_wait(200) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Service Offerings - driver.find_element_by_xpath(Global_Locators.serviceOfferings_xpath).click() - - #Select Disk offering - driver.find_element_by_xpath(Global_Locators.Offering_disk_xpath).click() - - # We will be searching for our disk offering into the table - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.Offering_table_xpath) # This returns a list of all Offerings in table - - for link in linkclass: - - if link.text == "Test Disk Name": - link.click() - - time.sleep(2) - - # Click Edit - driver.find_element_by_css_selector(Global_Locators.Offering_edit_css).click() - - #Change name - driver.find_element_by_name(Global_Locators.Offering_editname_name).clear() - driver.find_element_by_name(Global_Locators.Offering_editname_name).send_keys("Test Name") - - # Change Description - driver.find_element_by_name(Global_Locators.Offering_editdescription_name).clear() - driver.find_element_by_name(Global_Locators.Offering_editdescription_name).send_keys("Test Description") - - #Click Done - driver.find_element_by_css_selector(Global_Locators.Offering_editdone_css).click() - time.sleep(10) - - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - self.assertEqual([], self.verificationErrors) - - # Now we will find this offering and delete it!! - - - - - - -class Disk_offering_Delete(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_diskdelete(self): - - driver = self.driver - self.driver.implicitly_wait(200) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Service Offerings - driver.find_element_by_xpath(Global_Locators.serviceOfferings_xpath).click() - - #Select Disk offering - driver.find_element_by_xpath(Global_Locators.Offering_disk_xpath).click() - - ## Action part - # We will be searching for our disk offering into the table - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.Offering_table_xpath) # This returns a list of all Offerings in table - - for link in linkclass: - - if link.text == "Test Name": - link.click() - - time.sleep(2) - - # Click Delete - driver.find_element_by_css_selector(Global_Locators.Offering_delete_css).click() - time.sleep(2) - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - time.sleep(20) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - - - - - - -class Compute_offering_Add(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_computeadd(self): - - driver = self.driver - self.driver.implicitly_wait(200) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Service Offerings - driver.find_element_by_xpath(Global_Locators.serviceOfferings_xpath).click() - - #Select Compute offering - driver.find_element_by_xpath(Global_Locators.Offering_compute_xpath).click() - - ## Action part - - # Add offering - driver.find_element_by_xpath(Global_Locators.Offering_add_xpath).click() - - # Following do not have Global locators - driver.find_element_by_id("label_name").clear() - driver.find_element_by_id("label_name").send_keys("Test Compute Name") - driver.find_element_by_id("label_description").clear() - driver.find_element_by_id("label_description").send_keys("Test Compute Description") - driver.find_element_by_id("label_num_cpu_cores").clear() - driver.find_element_by_id("label_num_cpu_cores").send_keys("2") - driver.find_element_by_id("label_cpu_mhz").clear() - driver.find_element_by_id("label_cpu_mhz").send_keys("2000") - driver.find_element_by_id("label_memory_mb").clear() - driver.find_element_by_id("label_memory_mb").send_keys("2048") - driver.find_element_by_id("label_network_rate").clear() - driver.find_element_by_id("label_network_rate").send_keys("10") - driver.find_element_by_id("label_offer_ha").click() - driver.find_element_by_xpath("//button[@type='button']").click() - - time.sleep(2) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - - time.sleep(30) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - - - - - -class Compute_offering_Edit(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_computeedit(self): - - - driver = self.driver - self.driver.implicitly_wait(200) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - ## Action part - # Go to Service Offerings - driver.find_element_by_xpath(Global_Locators.serviceOfferings_xpath).click() - - #Select Compute offering - driver.find_element_by_xpath(Global_Locators.Offering_compute_xpath).click() - - # We will be searching for our disk offering into the table - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.Offering_table_xpath) # This returns a list of all Offerings in table - - for link in linkclass: - - if link.text == "Test Compute Name": - link.click() - - time.sleep(2) - - - # Click Edit - driver.find_element_by_css_selector(Global_Locators.Offering_edit_css).click() - - #Change name - driver.find_element_by_name(Global_Locators.Offering_editname_name).clear() - driver.find_element_by_name(Global_Locators.Offering_editname_name).send_keys("Test Name") - - # Change Description - driver.find_element_by_name(Global_Locators.Offering_editdescription_name).clear() - driver.find_element_by_name(Global_Locators.Offering_editdescription_name).send_keys("Test Description") - - #Click Done - driver.find_element_by_css_selector(Global_Locators.Offering_editdone_css).click() - time.sleep(10) - - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - self.assertEqual([], self.verificationErrors) - - - - - - -class Compute_offering_Delete(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_computedelete(self): - - - driver = self.driver - self.driver.implicitly_wait(200) - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Service Offerings - driver.find_element_by_xpath(Global_Locators.serviceOfferings_xpath).click() - - #Select Compute offering - driver.find_element_by_xpath(Global_Locators.Offering_compute_xpath).click() - - ## Action part - # We will be searching for our disk offering into the table - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.Offering_table_xpath) # This returns a list of all Offerings in table - - for link in linkclass: - - if link.text == "Test Name": - link.click() - - time.sleep(2) - - # Click Delete - - driver.find_element_by_css_selector(Global_Locators.Offering_deletecompute_css).click() - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - - time.sleep(20) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) diff --git a/test/selenium/smoke/TemplatesAndISO.py b/test/selenium/smoke/TemplatesAndISO.py deleted file mode 100644 index c69b4469116..00000000000 --- a/test/selenium/smoke/TemplatesAndISO.py +++ /dev/null @@ -1,244 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' -ISO PART YET TO BE ADDED:: remove this after adding it. -''' - -import sys, os -sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/'+'../lib')) - - - -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import Select -from selenium.common.exceptions import NoSuchElementException -import unittest, time -import initialize -import Global_Locators - - - - -class Template_Add(unittest.TestCase): - - - - def setUp(self): - - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_templateadd(self): - - - driver = self.driver - - ## Action part - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Templates - driver.find_element_by_xpath(Global_Locators.templates_xpath).click() - - #Select Template from drop down list - driver.find_element_by_xpath(Global_Locators.template_xpath).click() - - # Add Template - driver.find_element_by_xpath(Global_Locators.AddTemplate_xpath).click() - - # Following have names.. so they do not have their global entries. - driver.find_element_by_id("label_name").clear() - driver.find_element_by_id("label_name").send_keys("Test Template Ubuntu") - driver.find_element_by_id("label_description").clear() - driver.find_element_by_id("label_description").send_keys("Ubuntu 10.04") - driver.find_element_by_id("URL").clear() - driver.find_element_by_id("URL").send_keys("http://nfs1.lab.vmops.com/templates/Ubuntu/Ubuntuu-10-04-64bit-server.vhd") - Select(driver.find_element_by_id("label_os_type")).select_by_visible_text("Ubuntu 10.04 (64-bit)") - driver.find_element_by_id("label_public").click() - driver.find_element_by_id("label_featured").click() - driver.find_element_by_xpath("//button[@type='button']").click() - - time.sleep(2) - - # Go to Dash Board - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - - - time.sleep(600) - - ##Verification will be if this offering shows up into table and we can actually edit it. - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - - - - - -class Template_Edit(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_templateedit(self): - - driver = self.driver - - ## Action part - - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Templates - driver.find_element_by_xpath(Global_Locators.templates_xpath).click() - - #Select Template from drop down list - driver.find_element_by_xpath(Global_Locators.template_xpath).click() - - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.template_table_xpath) # This returns a list - - for link in linkclass: - - if link.text == "Test Template Ubuntu": # We will search for our VM in this table - link.click() - - time.sleep(2) - - # Change name - driver.find_element_by_name("name").clear() - driver.find_element_by_name("name").send_keys("Test template") - - - # Change Description - driver.find_element_by_name("displaytext").clear() - driver.find_element_by_name("displaytext").send_keys("ubuntu") - - driver.find_element_by_css_selector(Global_Locators.template_editdone_css).click() - time.sleep(2) - - #Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(10) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - -# Now we will find this offering and delete it!! - - - - - - -class Template_Delete(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - - def test_templatedelete(self): - - driver = self.driver - - ## Action part - #Make sure you are on Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - # Go to Templates - driver.find_element_by_xpath(Global_Locators.templates_xpath).click() - - #Select Template from drop down list - driver.find_element_by_xpath(Global_Locators.template_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.template_table_xpath) # This returns a list - - for link in linkclass: - - if link.text == "Test Template": # We will search for our VM in this table - link.click() - - time.sleep(2) - - driver.find_element_by_css_selector(Gloabl_Locators.template_delete_css).click() - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - - time.sleep(2) - - #Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - - time.sleep(20) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) diff --git a/test/selenium/smoke/VM_lifeCycle.py b/test/selenium/smoke/VM_lifeCycle.py deleted file mode 100644 index 480c45c8237..00000000000 --- a/test/selenium/smoke/VM_lifeCycle.py +++ /dev/null @@ -1,613 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import sys, os -sys.path.append(os.path.abspath(os.path.dirname(__file__) + '/'+'../lib')) - - - -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import Select -from selenium.common.exceptions import NoSuchElementException -import unittest, time -import initialize -import Global_Locators - - - -class deployVM(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_deployvm(self): - - - ## Action Part - # VM will be named Auto-VM and this VM will be used in all subsequent tests. - # Deploy an Instance named Auto-VM Default CentOS no GUI Template - - driver = self.driver - self.driver.implicitly_wait(30) - driver.refresh() ## Most Important step. Failure to do this will change XPATH location and Scripts will fail. - - - # Click on Instances link - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - # Click on add Instance on Instances page - driver.find_element_by_xpath(Global_Locators.add_instance_xpath).click() - - # Following select template action will fire automatically... ignore it. And leave following commented. - # driver.find_element_by_xpath("(//input[@name='select-template'])[3]").click() - #Click on Next button on Instances Wizard. - driver.find_element_by_xpath(Global_Locators.add_instance_next_xpath).click() - - # Nothing to do here as we will be using all default settings. (Default CentOS no GUI template should be highlighted here. Click Next - driver.find_element_by_xpath(Global_Locators.add_instance_next_xpath).click() - - # Nothing to do here. Medium Instance compute offering should be selected here. Click Next - driver.find_element_by_xpath(Global_Locators.add_instance_next_xpath).click() - - # Nothing to do here. Data Disk Offering : No Thanks!!. Click Next - driver.find_element_by_xpath(Global_Locators.add_instance_next_xpath).click() - - # Since this is our first instance; we must provide a network name. We will use Test-Network as out network name. - driver.find_element_by_xpath("(//input[@name='new-network-name'])[2]").click() - driver.find_element_by_xpath("(//input[@name='new-network-name'])[2]").clear() - driver.find_element_by_xpath("(//input[@name='new-network-name'])[2]").send_keys("Test-Network") - - #Click next - driver.find_element_by_xpath(Global_Locators.add_instance_next_xpath).click() - - # Give our VM a name here. Use Auto-VM as name - driver.find_element_by_xpath("(//input[@name='displayname'])[2]").click() - - driver.find_element_by_xpath("(//input[@name='displayname'])[2]").clear() - - driver.find_element_by_xpath("(//input[@name='displayname'])[2]").send_keys("Auto-VM") - - # All data filled. Click Launch VM. (It has the same xpath as Next button. So we will use Next Variable here. - driver.find_element_by_xpath(Global_Locators.add_instance_next_xpath).click() - - print '\n' + '\n' + "VM Deployment is complete... wait for 5 mins to check deployment status" + '\n' + '\n' - - - - ## Verification Part - - - ## Now we must wait for some random time (Educated guess based on experience) and check if VM has been deployed and if it is in running state. - ## Should take about 4 min to deploy VM.. but we will wait 5 mins and check the status , we will do this twice. So total 2 check within 10 mins with first check occuring at 5th min. - - - driver.refresh() # Refresh UI Page; This polls latest status. - - # Click on Instances link - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - count = 1 - - while (count > 0): - - time.sleep(300) - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - print "found VM in table .. checking status..." + '\n' + '\n' - link.click() - - status = driver.find_element_by_xpath(Global_Locators.state_xpath).text ## get the status of our VM - - if status == "Running" : - print "VM is in running state... continuing with other tests."+ '\n' + '\n' - break - else: - print "Need to check one more time after 5 mins" - continue - count = count - 1 - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - - def tearDown(self): - self.assertEqual([], self.verificationErrors) - - - - - -################################################################################################################################################################################################ - - - -class destroyVM(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_destroyvm(self): - - driver = self.driver - self.driver.implicitly_wait(100) - - ## Action part - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - time.sleep(2) - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - link.click() - - # Click on Destroy Instance button and confirm - time.sleep(2) - driver.find_element_by_css_selector(Global_Locators.destroyinstance_css).click() - time.sleep(2) - - # Click ok on confirmation - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - time.sleep(2) - - # Go to Dashboard - # driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - driver.refresh() - - ## Verification part - time.sleep(60) - - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - time.sleep(2) - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - link.click() - - - status = driver.find_element_by_xpath(Global_Locators.state_xpath).text ## get the status of our VM - if status == "Destroyed" : - print "VM is Destroyed...."+ '\n' + '\n' - else: - print "Something went wrong" - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - - - -################################################################################################################################################################################################ - - - - -class rebootVM(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_rebootvm(self): - - driver = self.driver - self.driver.implicitly_wait(30) - print "Verify this test manually for now" - - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(vmLifeAndNetwork.Server_Ip, username='root', password='password') - print '\n' + '\n' + "Before Reboot ...Executing command date ... " + '\n' + '\n' - stdin, stdout, stderr = ssh.exec_command('date') - print stdout.readlines() - print '\n' + '\n' + "Before Reboot ...Executing command last reboot | head -1 ..." + '\n' + '\n' - stdin, stdout, stderr = ssh.exec_command('last reboot | head -1') - print '\n' + '\n' + "Before Reboot ...Executing command uptime..." + '\n' + '\n' - stdin, stdout, stderr = ssh.exec_command('uptime') - print stdout.readlines() - ssh.close() - - - driver.refresh() - - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - count = 1 - - while (count > 0): - - #time.sleep(300) - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - print "found VM in table .. Rebooting now..." + '\n' + '\n' - link.click() - - driver.find_element_by_css_selector(Global_Locators.rebootinstance_css).click() - driver.find_element_by_xpath(Global_Locators.actionconfirm_xpath).click() - - # Sleep for 5 mins to ensure system gets rebooted. - time.sleep(300) - - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(vmLifeAndNetwork.Server_Ip, username='root', password='password') - print '\n' + '\n' + "After Reboot ...Executing command date ... " + '\n' + '\n' - stdin, stdout, stderr = ssh.exec_command('date') - print stdout.readlines() - print '\n' + '\n' + "After Reboot ...Executing command last reboot | head -1 ..." + '\n' + '\n' - stdin, stdout, stderr = ssh.exec_command('last reboot | head -1') - print '\n' + '\n' + "After Reboot ...Executing command uptime..." + '\n' + '\n' - stdin, stdout, stderr = ssh.exec_command('uptime') - print stdout.readlines() - ssh.close() - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - def tearDown(self): - self.assertEqual([], self.verificationErrors) - - -######################################################################################################################################################### - - - -class restoreVM(unittest.TestCase): - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_restorevm(self): - - driver = self.driver - self.driver.implicitly_wait(100) - - ## Action part - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - - link.click() - - # Click on Destroy Instance button and confirm - driver.find_element_by_css_selector(Global_Locators.restoreinstance_css).click() - - # Click ok on confirmation - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - - # Go to Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - - - ## Verification part - - time.sleep(60) - - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - link.click() - - - status = driver.find_element_by_xpath(Global_Locators.state_xpath).text ## get the status of our VM - - if status == "Stopped" : - print "VM is Restored. but in stopped state.. will start now."+ '\n' + '\n' - - else: - print "Something went wrong" - - - - - #VM will be in stop state so we must start it now - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - link.click() - - # Click on Start Instance. - driver.find_element_by_css_selector(Global_Locators.startinstance_css).click() - time.sleep(2) - - # Dismiss confirmation by clicking Yes - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - time.sleep(2) - - # Go to Dashboard - driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - time.sleep(2) - - print "VM is Started."+ '\n' + '\n' - - # status = None - time.sleep(60) - - # Dismiss the Start Instance information box. - driver.find_element_by_xpath(Global_Locators.actionconfirm_xpath).click() - time.sleep(2) - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - -######################################################################################################################################################### - - - -class startVM(unittest.TestCase): - - - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_startvm(self): - - driver = self.driver - self.driver.implicitly_wait(100) - - ## Action part - #driver.refresh() ## Most Important step. Failure to do this will change XPATH location and Scripts will fail. - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - print "found VM in table .. checking status..." + '\n' + '\n' - link.click() - - - - # Click on Start Instance. - driver.find_element_by_css_selector(Global_Locators.startinstance_css).click() - time.sleep(2) - - # Dismiss confirmation by clicking Yes - driver.find_element_by_xpath(Global_Locators.yesconfirmation_xapth).click() - time.sleep(2) - - # Go to Dashboard - #driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - driver.refresh() - - - ## Verification part - # status = None - time.sleep(60) - - # Dismiss the Start Instance information box. - driver.find_element_by_xpath(Global_Locators.actionconfirm_xpath).click() - time.sleep(2) - - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - time.sleep(2) - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - link.click() - - - status = driver.find_element_by_xpath(Global_Locators.state_xpath).text ## get the status of our VM - - if status == "Running" : - print "VM is in Running state..."+ '\n' + '\n' - - else: - print "Something went wrong" - - # Go to Dashboard - driver.refresh() - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - - - -######################################################################################################################################################### - - - -class stopVM(unittest.TestCase): - - def setUp(self): - - self.driver = initialize.getOrCreateWebdriver() - self.verificationErrors = [] - - - def test_stopvm(self): - - driver = self.driver - self.driver.implicitly_wait(100) - - ## Action part - driver.refresh() ## Important step. - - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - print "found VM in table .. checking status..." + '\n' + '\n' - link.click() - - - # HWe are on our VM information page. - driver.find_element_by_css_selector(Global_Locators.stopinstance_css).click() - time.sleep(2) - - # a Pop up must appear; below we will check the force stop check box and then we will click ok. - driver.find_element_by_id(Global_Locators.stopinstanceforce_id).click() - driver.find_element_by_xpath(Global_Locators.actionconfirm_xpath).click() - time.sleep(2) - - # Go to Dahsboard - #driver.find_element_by_xpath(Global_Locators.dashboard_xpath).click() - driver.refresh() - - # Should take less than min to stop the instance. We will check twice at interval of 45 seconds o be safe. - ## Verification part - time.sleep(60) - - # Click on Instances link and find our instance - driver.find_element_by_xpath(Global_Locators.instances_xpath).click() - - linkclass = None - linkclass = driver.find_elements_by_xpath(Global_Locators.instances_table_xpath) # This returns a list of all VM names in tables - - for link in linkclass: - - if link.text == "Auto-VM": # We will search for our VM in this table - link.click() - - - status = driver.find_element_by_xpath(Global_Locators.state_xpath).text ## get the status of our VM - - if status == "Stopped" : - print "VM is in Stopped state...."+ '\n' + '\n' - else: - print "Something went wrong" - - - - def is_element_present(self, how, what): - - try: self.driver.find_element(by=how, value=what) - except NoSuchElementException, e: return False - return True - - - - def tearDown(self): - - self.assertEqual([], self.verificationErrors) - - -######################################################################################################################################################### diff --git a/test/selenium/smoke/main.py b/test/selenium/smoke/main.py deleted file mode 100644 index c24d6700057..00000000000 --- a/test/selenium/smoke/main.py +++ /dev/null @@ -1,142 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import unittest -import HTMLTestRunner -import xmlrunner - - -global DRIVER -global MS_ip - -# Import test cases - -################################## -from Login_and_Accounts import * -from Service_Offering import * - -from TemplatesAndISO import * -from VM_lifeCycle import * - -################################### - - -# Following are BVT Tests -# serialize the test cases - - -suite = unittest.TestSuite() # setup new test suite - - -#################################################################################################### - -# Following logs admin user in and creates test account then logs admin user out and logs in as test to run tests. -# You should leave this as is for all the tests. - -suite.addTest(unittest.makeSuite(login)) #Login Admin - -time.sleep(5) -suite.addTest(unittest.makeSuite(createAcc)) # Create an Account test. We will use test account for all our tests - -time.sleep(5) -suite.addTest(unittest.makeSuite(logout)) #Logout Admin - -time.sleep(5) -suite.addTest(unittest.makeSuite(login_test)) # Login Test - - - -#################################################################################################### - - - -time.sleep(5) -suite.addTest(unittest.makeSuite(Disk_offering_Add)) - -time.sleep(5) -suite.addTest(unittest.makeSuite(Disk_offering_Edit)) - -time.sleep(5) -suite.addTest(unittest.makeSuite(Disk_offering_Delete)) - -time.sleep(5) -suite.addTest(unittest.makeSuite(Compute_offering_Add)) - -time.sleep(5) -suite.addTest(unittest.makeSuite(Compute_offering_Edit)) - -time.sleep(5) -suite.addTest(unittest.makeSuite(Compute_offering_Delete)) - - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(deployVM)) - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(stopVM)) - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(startVM)) - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(destroyVM)) - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(restoreVM)) - - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(Template_Add)) - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(Template_Edit)) - -# time.sleep(5) -# suite.addTest(unittest.makeSuite(Template_Delete)) - - -#################################################################################################### - -# Following logs test user out and logs back in as Admin and tears down the test account. -# You should leave this as is for all the tests. - -suite.addTest(unittest.makeSuite(logout)) #Logout test -time.sleep(5) -suite.addTest(unittest.makeSuite(login)) #Login Admin -time.sleep(5) -suite.addTest(unittest.makeSuite(tearAcc)) # Delete Account test - -#################################################################################################### - - - -# If XML reports compatible with junit's XML output are desired then leave folowing code as is. -# If HTML reports are desired follow instructions - - -#Comment following line for HTML and uncomment for XML -runner = xmlrunner.XMLTestRunner(output='test-reports') - -#Comment following line for XML and uncomment for HTML -#runner = HTMLTestRunner.HTMLTestRunner() - -#header is required for displaying the website -#Comment following line for XML and uncomment for HTML -#print "content-type: text/html\n" - -# Leave following as is for either XML or HTML -runner.run(suite) diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 3248016528a..8fafd841be4 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -257,6 +257,8 @@ known_categories = { 'deleteASNRange': 'AS Number Range', 'listASNumbers': 'AS Number', 'releaseASNumber': 'AS Number', + 'addNodesToKubernetesCluster': 'Kubernetes Service', + 'removeNodesFromKubernetesCluster': 'Kubernetes Service', 'configureStorageAccess': 'Storage Access Groups', 'listStorageAccessGroups': 'Storage Access Groups' } diff --git a/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json b/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json new file mode 100644 index 00000000000..edaa11f96ce --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/cks-ubuntu-2204.json @@ -0,0 +1,56 @@ +{ + "_license": "Apache License 2.0", + "builders": [ + { + "accelerator": "kvm", + "boot_command": [ + "clinux /casper/vmlinuz --- autoinstall ds='nocloud-net;seedfrom=http://{{ .HTTPIP }}:{{ .HTTPPort }}/'", + "", + "initrd /casper/initrd", + "", + "boot", + "" + ], + "vm_name": "cks-ubuntu-2204", + "iso_checksum": "sha256:5e38b55d57d94ff029719342357325ed3bda38fa80054f9330dc789cd2d43931", + "iso_url": "https://old-releases.ubuntu.com/releases/jammy/ubuntu-22.04.2-live-server-amd64.iso", + "shutdown_command": "sudo shutdown -P now", + "net_device": "virtio-net", + "output_directory": "../dist", + "format": "qcow2", + "headless": true, + "http_directory": "http", + "ssh_password": "cloud", + "ssh_timeout": "30m", + "ssh_username": "cloud", + "type": "qemu", + "disk_interface": "virtio", + "disk_size": "5000M", + "qemuargs": [ + [ + "-m", + "2048M" + ], + [ + "-smp", + "1" + ] + ] + } + ], + "description": "CloudStack SystemVM template", + "provisioners": [ + { + "execute_command": "echo 'cloud' | sudo -u root -S bash {{.Path}}", + "scripts": [ + "scripts/apt_upgrade.sh", + "scripts/configure_networking.sh", + "scripts/configure-cloud-init.sh", + "scripts/setup-interfaces.sh", + "scripts/add-interface-rule.sh", + "scripts/cleanup.sh" + ], + "type": "shell" + } + ] +} diff --git a/test/selenium/browser/__init__.py b/tools/appliance/cks/ubuntu/22.04/http/meta-data similarity index 100% rename from test/selenium/browser/__init__.py rename to tools/appliance/cks/ubuntu/22.04/http/meta-data diff --git a/tools/appliance/cks/ubuntu/22.04/http/user-data b/tools/appliance/cks/ubuntu/22.04/http/user-data new file mode 100644 index 00000000000..15a7f8f3235 --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/http/user-data @@ -0,0 +1,103 @@ +#cloud-config +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +autoinstall: + version: 1 + # Disable ssh server during installation, otherwise packer tries to connect and exceed max attempts + early-commands: + - systemctl stop ssh + # Configure the locale + locale: en_US + keyboard: + layout: us + refresh-installer: + update: yes + channel: stable + # Create a single-partition with no swap space. Kubernetes + # really dislikes the idea of anyone else managing memory. + # For more information on how partitioning is configured, + # please refer to https://curtin.readthedocs.io/en/latest/topics/storage.html. + storage: + swap: + size: 0 + grub: + replace_linux_default: false + config: + - type: disk + id: disk-0 + size: smallest + grub_device: true + preserve: false + ptable: msdos + wipe: superblock + - type: partition + id: partition-0 + device: disk-0 + size: -1 + number: 1 + preserve: false + flag: boot + - type: format + id: format-0 + volume: partition-0 + fstype: ext4 + preserve: false + - type: mount + id: mount-0 + device: format-0 + path: / + updates: 'all' + ssh: + install-server: true + allow-pw: true + # Customize the list of packages installed. + packages: + - open-vm-tools + - openssh-server + - cloud-init + - wget + - tasksel + # Create the default user. + # Ensures the "cloud" user doesn't require a password to use sudo. + user-data: + disable_root: false + timezone: UTC + users: + - name: cloud + # openssl passwd -6 -stdin <<< cloud + passwd: $6$pAFEBhaCDzN4ZmrO$kMmUuxhPMx447lJ8Mtas8n6uqkojh94nQ7I8poI6Kl4vRGeZKE57utub1cudS1fGyG8HUxK9YHIygd7vCpRFN0 + groups: [adm, cdrom, dip, plugdev, lxd, sudo] + lock-passwd: false + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + + # This command runs after all other steps; it: + # 1. Disables swapfiles + # 2. Removes the existing swapfile + # 3. Removes the swapfile entry from /etc/fstab + # 4. Removes snapd, https://bugs.launchpad.net/subiquity/+bug/1946609 + # 5. Cleans up any packages that are no longer required + # 6. Removes the cached list of packages + late-commands: + - curtin in-target --target=/target -- swapoff -a + - curtin in-target --target=/target -- rm -f /swap.img + - curtin in-target --target=/target -- sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab + - chroot /target apt-get purge -y snapd + - curtin in-target --target=/target -- apt-get purge --auto-remove -y + - curtin in-target --target=/target -- apt-get clean + - curtin in-target --target=/target -- rm -rf /var/lib/apt/lists/* diff --git a/test/selenium/common/__init__.py b/tools/appliance/cks/ubuntu/22.04/scripts/add-interface-rule.sh similarity index 50% rename from test/selenium/common/__init__.py rename to tools/appliance/cks/ubuntu/22.04/scripts/add-interface-rule.sh index 13a83393a91..7a28f0e55cb 100644 --- a/test/selenium/common/__init__.py +++ b/tools/appliance/cks/ubuntu/22.04/scripts/add-interface-rule.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,3 +15,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. + +# File and rule definition +RULE_FILE="/etc/udev/rules.d/90-new-interface.rules" +RULE='ACTION=="add|change|remove", SUBSYSTEM=="net", DRIVERS=="?*", RUN+="/bin/systemctl --no-block start update-netplan.service"' + +# Ensure the file exists, or create it +if [[ ! -f $RULE_FILE ]]; then + touch "$RULE_FILE" + echo "Created $RULE_FILE." +fi + +# Check if the rule already exists to prevent duplication +if grep -Fxq "$RULE" "$RULE_FILE"; then + echo "Rule already exists in $RULE_FILE." +else + # Add the rule to the file + echo "$RULE" | tee -a "$RULE_FILE" > /dev/null + echo "Rule added to $RULE_FILE." +fi + +# Reload udev rules and apply the changes +udevadm control --reload-rules +udevadm trigger +echo "Udev rules reloaded and triggered." diff --git a/test/selenium/cspages/cspage.py b/tools/appliance/cks/ubuntu/22.04/scripts/apt_upgrade.sh similarity index 71% rename from test/selenium/cspages/cspage.py rename to tools/appliance/cks/ubuntu/22.04/scripts/apt_upgrade.sh index 4f52e4d116d..22d25d628ef 100644 --- a/test/selenium/cspages/cspage.py +++ b/tools/appliance/cks/ubuntu/22.04/scripts/apt_upgrade.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,6 +16,22 @@ # specific language governing permissions and limitations # under the License. -class CloudStackPage(object): - def __init__(): - self.browser = None +set -e +set -x + +function apt_upgrade() { + DEBIAN_FRONTEND=noninteractive + DEBIAN_PRIORITY=critical + + rm -fv /root/*.iso + apt-get -q -y update + + apt-get -q -y upgrade + apt-get -q -y dist-upgrade + + apt-get -y autoremove --purge + apt-get autoclean + apt-get clean +} + +return 2>/dev/null || apt_upgrade diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/cleanup.sh b/tools/appliance/cks/ubuntu/22.04/scripts/cleanup.sh new file mode 100644 index 00000000000..ab0ceb62861 --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/cleanup.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e + +function cleanup_apt() { + export DEBIAN_FRONTEND=noninteractive + apt-get -y remove --purge dictionaries-common busybox \ + task-english task-ssh-server tasksel tasksel-data laptop-detect wamerican sharutils \ + nano util-linux-locales krb5-locales + + apt-get -y autoremove --purge + apt-get autoclean + apt-get clean +} + +# Removing leftover leases and persistent rules +function cleanup_dhcp() { + rm -f /var/lib/dhcp/* +} + +# Make sure Udev doesn't block our network +function cleanup_dev() { + echo "cleaning up udev rules" + rm -f /etc/udev/rules.d/70-persistent-net.rules + rm -rf /dev/.udev/ + rm -f /lib/udev/rules.d/75-persistent-net-generator.rules +} + +function cleanup_misc() { + # Scripts + rm -fr /home/cloud/cloud_scripts* + rm -f /usr/share/cloud/cloud-scripts.tar + rm -f /root/.rnd + rm -f /var/www/html/index.html + # Logs + rm -f /var/log/*.log + rm -f /var/log/apache2/* + rm -f /var/log/messages + rm -f /var/log/syslog + rm -f /var/log/messages + rm -fr /var/log/apt + rm -fr /var/log/installer + # Docs and data files + rm -fr /var/lib/apt/* + rm -fr /var/cache/apt/* + rm -fr /var/cache/debconf/*old + rm -fr /usr/share/doc + rm -fr /usr/share/man + rm -fr /usr/share/info + rm -fr /usr/share/lintian + rm -fr /usr/share/apache2/icons + find /usr/share/locale -type f | grep -v en_US | xargs rm -fr + find /usr/share/zoneinfo -type f | grep -v UTC | xargs rm -fr + rm -fr /tmp/* +} + +function cleanup() { + cleanup_apt + cleanup_dhcp + cleanup_dev + cleanup_misc +} + +return 2>/dev/null || cleanup diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh b/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh new file mode 100644 index 00000000000..4e4979c936f --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/configure-cloud-init.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +function install_packages() { + apt-get install -y qemu-guest-agent rsyslog logrotate cron net-tools ifupdown cloud-guest-utils conntrack apt-transport-https ca-certificates curl \ + gnupg gnupg-agent software-properties-common gnupg lsb-release + apt-get install -y python3-json-pointer python3-jsonschema cloud-init resolvconf + + sudo mkdir -p /etc/apt/keyrings + echo "Creating /opt/bin directory" + sudo mkdir -p /opt/bin + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + apt update + apt install containerd.io + + systemctl start containerd + systemctl enable containerd +} + +function configure_services() { + install_packages + + systemctl daemon-reload +cat < /etc/cloud/cloud.cfg.d/cloudstack.cfg +datasource_list: ['CloudStack'] +datasource: + CloudStack: + max_wait: 120 + timeout: 50 +EOF +} + +configure_services diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/configure_networking.sh b/tools/appliance/cks/ubuntu/22.04/scripts/configure_networking.sh new file mode 100644 index 00000000000..a5e4179a441 --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/configure_networking.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +set -x + +HOSTNAME=cksnode + +function configure_resolv_conf() { + grep 8.8.8.8 /etc/resolv.conf && grep 8.8.4.4 /etc/resolv.conf && return + + cat > /etc/resolv.conf << EOF +nameserver 8.8.8.8 +nameserver 8.8.4.4 +EOF +} + +# Delete entry in /etc/hosts derived from dhcp +function delete_dhcp_ip() { + result=$(grep 127.0.1.1 /etc/hosts || true) + [ "${result}" == "" ] && return + + sed -i '/127.0.1.1/d' /etc/hosts +} + +function configure_hostname() { + sed -i "s/root@\(.*\)$/root@$HOSTNAME/g" /etc/ssh/ssh_host_*.pub + + echo "$HOSTNAME" > /etc/hostname + hostname $HOSTNAME +} + +function configure_interfaces() { + cat > /etc/network/interfaces << EOF +source /etc/network/interfaces.d/* + +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface +auto ens35 +iface ens35 inet dhcp + +EOF + +echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf +sysctl -p /etc/sysctl.conf +} + +function configure_networking() { + configure_interfaces + configure_resolv_conf + delete_dhcp_ip + configure_hostname +} + +return 2>/dev/null || configure_networking diff --git a/tools/appliance/cks/ubuntu/22.04/scripts/setup-interfaces.sh b/tools/appliance/cks/ubuntu/22.04/scripts/setup-interfaces.sh new file mode 100644 index 00000000000..4dd2caead56 --- /dev/null +++ b/tools/appliance/cks/ubuntu/22.04/scripts/setup-interfaces.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Create the script in the /opt/bin directory +SCRIPT_PATH="/usr/local/bin/update-netplan.sh" + +cat <<'EOF' > $SCRIPT_PATH +#!/bin/bash + +echo "New interface detected: $INTERFACE" >> /var/log/interface-events.log +CONFIG_FILE="/etc/netplan/config.yaml" + +# Generate a new netplan configuration +echo "network:" > $CONFIG_FILE +echo " ethernets:" >> $CONFIG_FILE + +# Loop through all available interfaces +for iface in $(ls /sys/class/net | grep -vE '^lo$'); do +cat <> $CONFIG_FILE + $iface: + dhcp4: true +EOL +done + +chmod 600 $CONFIG_FILE + +netplan apply +EOF + +tee /etc/systemd/system/update-netplan.service < ~/.ssh/authorized_keys +else + echo "Please place Management server public key in the variables" + exit 1 +fi diff --git a/tools/appliance/cks/ubuntu/build.sh b/tools/appliance/cks/ubuntu/build.sh new file mode 100755 index 00000000000..c5866843478 --- /dev/null +++ b/tools/appliance/cks/ubuntu/build.sh @@ -0,0 +1,346 @@ +#!/bin/bash -l +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# build script which wraps around packer and virtualbox to create the CKS template + +function usage() { + cat <&2 +} + +function error() { + log ERROR $@ + exit 1 +} + +# cleanup code support +declare -a on_exit_items + +function on_exit() { + for (( i=${#on_exit_items[@]}-1 ; i>=0 ; i-- )) ; do + sleep 2 + log DEBUG "on_exit: ${on_exit_items[i]}" + eval ${on_exit_items[i]} + done +} + +function add_on_exit() { + local n=${#on_exit_items[*]} + on_exit_items[${n}]="$*" + if [ ${n} -eq 0 ]; then + log DEBUG "Setting trap" + trap on_exit EXIT + fi +} + +# retry code support +function retry() { + local times=$1 + shift + local count=0 + while [ ${count} -lt ${times} ]; do + "$@" && break + count=$(( $count + 1 )) + sleep ${count} + done + + if [ ${count} -eq ${times} ]; then + error "Failed ${times} times: $@" + fi +} + +### +### Script logic +### + +function prepare() { + log INFO "preparing for build" + rm -rf dist *.ova *.vhd *.vdi *.qcow* *.bz2 *.vmdk *.ovf +} + +function packer_build() { + log INFO "building new image with packer" + #cd ${appliance_build_name} && packer build template.json && cd .. + cd 22.04 && packer build ${appliance_build_name}.json && cd .. +} + +function stage_vmx() { + cat << VMXFILE > "${1}.vmx" +.encoding = "UTF-8" +displayname = "${1}" +annotation = "${1}" +guestos = "otherlinux-64" +virtualHW.version = "11" +config.version = "8" +numvcpus = "1" +cpuid.coresPerSocket = "1" +memsize = "256" +pciBridge0.present = "TRUE" +pciBridge4.present = "TRUE" +pciBridge4.virtualDev = "pcieRootPort" +pciBridge4.functions = "8" +pciBridge5.present = "TRUE" +pciBridge5.virtualDev = "pcieRootPort" +pciBridge5.functions = "8" +pciBridge6.present = "TRUE" +pciBridge6.virtualDev = "pcieRootPort" +pciBridge6.functions = "8" +pciBridge7.present = "TRUE" +pciBridge7.virtualDev = "pcieRootPort" +pciBridge7.functions = "8" +vmci0.present = "TRUE" +floppy0.present = "FALSE" +ide0:0.clientDevice = "FALSE" +ide0:0.present = "TRUE" +ide0:0.deviceType = "atapi-cdrom" +ide0:0.autodetect = "TRUE" +ide0:0.startConnected = "FALSE" +mks.enable3d = "false" +svga.autodetect = "false" +svga.vramSize = "4194304" +scsi0:0.present = "TRUE" +scsi0:0.deviceType = "disk" +scsi0:0.fileName = "$2" +scsi0:0.mode = "persistent" +scsi0:0.writeThrough = "false" +scsi0.virtualDev = "lsilogic" +scsi0.present = "TRUE" +vmci0.unrestricted = "false" +vcpu.hotadd = "false" +vcpu.hotremove = "false" +firmware = "bios" +mem.hotadd = "false" +VMXFILE +} + +function xen_server_export() { + log INFO "creating xen server export" + set +e + which faketime >/dev/null 2>&1 && which vhd-util >/dev/null 2>&1 + local result=$? + set -e + if [ ${result} == 0 ]; then + qemu-img convert -f qcow2 -O raw "dist/${appliance}" img.raw + vhd-util convert -s 0 -t 1 -i img.raw -o stagefixed.vhd + faketime '2010-01-01' vhd-util convert -s 1 -t 2 -i stagefixed.vhd -o "${appliance_build_name}-xen.vhd" + rm -f *.bak + bzip2 "${appliance_build_name}-xen.vhd" + mv "${appliance_build_name}-xen.vhd.bz2" dist/ + log INFO "${appliance} exported for XenServer: dist/${appliance_build_name}-xen.vhd.bz2" + else + log WARN "** Skipping ${appliance_build_name} export for XenServer: faketime or vhd-util command is missing. **" + log WARN "** faketime source code is available from https://github.com/wolfcw/libfaketime **" + fi +} + +function ovm_export() { + log INFO "creating OVM export" + qemu-img convert -f qcow2 -O raw "dist/${appliance}" "dist/${appliance_build_name}-ovm.raw" + cd dist && bzip2 "${appliance_build_name}-ovm.raw" && cd .. + log INFO "${appliance} exported for OracleVM: dist/${appliance_build_name}-ovm.raw.bz2" +} + +function kvm_export() { + log INFO "creating kvm export" + set +e + qemu-img convert -o compat=0.10 -f qcow2 -c -O qcow2 "dist/${appliance}" "dist/${appliance_build_name}-kvm.qcow2" + local qemuresult=$? + cd dist && bzip2 "${appliance_build_name}-kvm.qcow2" && cd .. + log INFO "${appliance} exported for KVM: dist/${appliance_build_name}-kvm.qcow2.bz2" +} + +function vmware_export() { + log INFO "creating vmware export" + qemu-img convert -f qcow2 -O vmdk "dist/${appliance}" "dist/${appliance_build_name}-vmware.vmdk" + + if ! ovftool_loc="$(type -p "ovftool")" || [ -z "$ovftool_loc" ]; then + log INFO "ovftool not found, skipping ova generation for VMware" + return + fi + + log INFO "ovftool found, using it to export ova file" + CDIR=$PWD + cd dist + chmod 666 ${appliance_build_name}-vmware.vmdk + stage_vmx ${appliance_build_name}-vmware ${appliance_build_name}-vmware.vmdk + ovftool ${appliance_build_name}-vmware.vmx ${appliance_build_name}-vmware.ova + rm -f *vmx *vmdk + cd $CDIR + log INFO "${appliance} exported for VMWare: dist/${appliance_build_name}-vmware.ova" +} + +function hyperv_export() { + log INFO "creating hyperv export" + qemu-img convert -f qcow2 -O vpc "dist/${appliance}" "dist/${appliance_build_name}-hyperv.vhd" + CDIR=$PWD + cd dist + zip "${appliance_build_name}-hyperv.vhd.zip" "${appliance_build_name}-hyperv.vhd" + rm -f *vhd + cd $CDIR + log INFO "${appliance} exported for HyperV: dist/${appliance_build_name}-hyperv.vhd.zip" +} + +### +### Main invocation +### + +function main() { + prepare + + packer_build + + # process the disk at dist + kvm_export + ovm_export + xen_server_export + vmware_export + hyperv_export + rm -f "dist/${appliance}" + cd dist && chmod +r * && cd .. + cd dist && md5sum * > md5sum.txt && cd .. + cd dist && sha512sum * > sha512sum.txt && cd .. + add_on_exit log INFO "BUILD SUCCESSFUL" +} + +# we only run main() if not source-d +return 2>/dev/null || main diff --git a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json index 2165b571576..a3507c3a091 100644 --- a/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json @@ -32,8 +32,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:022895e699231c94abf7012f86cabc587dc576f07f856c87609d5d40c1f921d805a5a862cba94c1a47d09aaa565ec445222e338e73d1fa1affc4fc5908bb50ad", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.10.0/arm64/iso-cd/debian-12.10.0-arm64-netinst.iso", + "iso_checksum": "sha512:892cf1185a214d16ff62a18c6b89cdcd58719647c99916f6214bfca6f9915275d727b666c0b8fbf022c425ef18647e9759974abf7fc440431c39b50c296a98d3", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.11.0/arm64/iso-cd/debian-12.11.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json index 7f1b5c4befa..0ee62f1fd4b 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json @@ -31,8 +31,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:022895e699231c94abf7012f86cabc587dc576f07f856c87609d5d40c1f921d805a5a862cba94c1a47d09aaa565ec445222e338e73d1fa1affc4fc5908bb50ad", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.10.0/arm64/iso-cd/debian-12.10.0-arm64-netinst.iso", + "iso_checksum": "sha512:892cf1185a214d16ff62a18c6b89cdcd58719647c99916f6214bfca6f9915275d727b666c0b8fbf022c425ef18647e9759974abf7fc440431c39b50c296a98d3", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.11.0/arm64/iso-cd/debian-12.11.0-arm64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemu_binary": "qemu-system-aarch64", diff --git a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json index d74c408cbc3..94d2f5172f7 100644 --- a/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json +++ b/tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json @@ -27,8 +27,8 @@ "format": "qcow2", "headless": true, "http_directory": "http", - "iso_checksum": "sha512:cb089def0684fd93c9c2fbe45fd16ecc809c949a6fd0c91ee199faefe7d4b82b64658a264a13109d59f1a40ac3080be2f7bd3d8bf3e9cdf509add6d72576a79b", - "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.10.0/amd64/iso-cd/debian-12.10.0-amd64-netinst.iso", + "iso_checksum": "sha512:0921d8b297c63ac458d8a06f87cd4c353f751eb5fe30fd0d839ca09c0833d1d9934b02ee14bbd0c0ec4f8917dde793957801ae1af3c8122cdf28dde8f3c3e0da", + "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.11.0/amd64/iso-cd/debian-12.11.0-amd64-netinst.iso", "net_device": "virtio-net", "output_directory": "../dist", "qemuargs": [ diff --git a/tools/docker/Dockerfile.smokedev b/tools/docker/Dockerfile.smokedev index f929294c2ce..b4c55b30967 100644 --- a/tools/docker/Dockerfile.smokedev +++ b/tools/docker/Dockerfile.smokedev @@ -88,7 +88,6 @@ COPY test/conf /root/test/conf COPY test/metadata /root/test/metadata COPY test/pom.xml /root/test/pom.xml COPY test/scripts /root/test/scripts -COPY test/selenium /root/test/selenium COPY test/systemvm /root/test/systemvm COPY test/target /root/test/target COPY tools/pom.xml /root/tools/pom.xml diff --git a/ui/package-lock.json b/ui/package-lock.json index f3087d1bf98..c9f90b6552a 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -2816,6 +2816,12 @@ "@types/node": "*" } }, + "@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "optional": true + }, "@types/uglify-js": { "version": "3.16.0", "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.16.0.tgz", @@ -9035,6 +9041,14 @@ } } }, + "dompurify": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.6.tgz", + "integrity": "sha512-/2GogDQlohXPZe6D6NOgQvXLPSYBqIWMnZ8zzOhn09REE4eyAzb+Hed3jhoM9OkuaJ8P6ZGTTVWQKAi8ieIzfQ==", + "requires": { + "@types/trusted-types": "^2.0.7" + } + }, "domutils": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", diff --git a/ui/package.json b/ui/package.json index df8c5d5f82b..3c7e1299a9d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -49,6 +49,7 @@ "chartjs-adapter-moment": "^1.0.0", "core-js": "^3.21.1", "cronstrue": "^2.26.0", + "dompurify": "^3.2.6", "enquire.js": "^2.1.6", "js-cookie": "^2.2.1", "lodash": "^4.17.15", diff --git a/ui/public/config.json b/ui/public/config.json index 38d1fd9bbe7..d56e6f03783 100644 --- a/ui/public/config.json +++ b/ui/public/config.json @@ -97,5 +97,18 @@ "basicZoneEnabled": true, "multipleServer": false, "allowSettingTheme": true, - "docHelpMappings": {} + "imageSelectionInterface": "modern", + "showUserCategoryForModernImageSelection": true, + "showAllCategoryForModernImageSelection": false, + "docHelpMappings": {}, + "announcementBanner": { + "enabled": false, + "showIcon": false, + "closable": true, + "persistDismissal": true, + "type": "info", + "message": "🤔 Sample Announcement: New Feature Available: Check out our latest dashboard improvements! Learn more", + "startDate": "2025-06-01T00:00:00Z", + "endDate": "2025-07-16T00:00:00Z" + } } diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index b3d5536e1f1..b649f6914bb 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -53,6 +53,8 @@ "label.acquiring.ip": "Acquiring IP", "label.associated.resource": "Associated resource", "label.action": "Action", +"label.action.add.nodes.to.kubernetes.cluster": "Add nodes to Kubernetes cluster", +"label.action.remove.nodes.from.kubernetes.cluster": "Remove nodes from Kubernetes cluster", "label.action.attach.disk": "Attach disk", "label.action.attach.iso": "Attach ISO", "label.action.attach.to.instance": "Attach to Instance", @@ -87,8 +89,9 @@ "label.action.delete.egress.firewall": "Delete egress firewall rule", "label.action.delete.firewall": "Delete firewall rule", "label.action.delete.interface.static.route": "Remove Tungsten Fabric interface static route", -"label.action.delete.guest.os": "Delete guest os", -"label.action.delete.guest.os.hypervisor.mapping": "Delete guest os hypervisor mapping", +"label.action.delete.guest.os": "Delete guest OS", +"label.action.delete.guest.os.category": "Delete guest OS category", +"label.action.delete.guest.os.hypervisor.mapping": "Delete guest OS hypervisor mapping", "label.action.delete.ip.range": "Delete IP range", "label.action.delete.iso": "Delete ISO", "label.action.delete.load.balancer": "Delete load balancer rule", @@ -263,8 +266,9 @@ "label.add.firewall": "Add firewall rule", "label.add.firewallrule": "Add Firewall Rule", "label.add.guest.network": "Add guest Network", -"label.add.guest.os": "Add guest os", -"label.add.guest.os.hypervisor.mapping": "Add guest os hypervisor mapping", +"label.add.guest.os": "Add guest OS", +"label.add.guest.os.category": "Add guest OS category", +"label.add.guest.os.hypervisor.mapping": "Add guest OS hypervisor mapping", "label.add.host": "Add host", "label.add.ingress.rule": "Add ingress rule", "label.add.intermediate.certificate": "Add intermediate certificate", @@ -278,6 +282,7 @@ "label.add.list.name": "ACL List name", "label.add.logical.router": "Add Logical Router to this Network", "label.add.more": "Add more", +"label.add.nodes": "Add Nodes to Kubernetes Cluster", "label.add.netscaler.device": "Add Netscaler device", "label.add.network": "Add Network", "label.add.network.acl": "Add Network ACL", @@ -516,9 +521,17 @@ "label.cisco.nexus1000v.password": "Nexus 1000v password", "label.cisco.nexus1000v.username": "Nexus 1000v username", "label.cks.cluster.autoscalingenabled": "Enable auto scaling on this cluster", +"label.cks.cluster.control.nodes.offeringid": "Service Offering for Control Nodes", +"label.cks.cluster.control.nodes.templateid": "Template for Control Nodes", +"label.cks.cluster.etcd.nodes": "Etcd Nodes", +"label.cks.cluster.etcd.nodes.offeringid": "Service Offering for etcd Nodes", +"label.cks.cluster.etcd.nodes.templateid": "Template for etcd Nodes", "label.cks.cluster.maxsize": "Maximum cluster size (Worker nodes)", "label.cks.cluster.minsize": "Minimum cluster size (Worker nodes)", +"label.cks.cluster.node.manual.upgrade": "Mark nodes for manual upgrade", "label.cks.cluster.size": "Cluster size (Worker nodes)", +"label.cks.cluster.worker.nodes.offeringid": "Service Offering for Worker Nodes", +"label.cks.cluster.worker.nodes.templateid": "Template for Worker Nodes", "label.cleanup": "Clean up", "label.clear": "Clear", "label.clear.list": "Clear list", @@ -1021,9 +1034,10 @@ "label.fix.errors": "Fix errors", "label.fixed": "Fixed offering", "label.for": "for", +"label.forcks": "For CKS", "label.forbidden": "Forbidden", "label.forced": "Force", -"label.force.ms.to.import.vm.files": "Force MS to export OVF from VMware to temporary storage", +"label.force.ms.to.import.vm.files": "Enable to force OVF Download via Management Server. Disable to use KVM Host ovftool (if installed)", "label.force.stop": "Force stop", "label.force.reboot": "Force reboot", "label.forceencap": "Force UDP encapsulation of ESP packets", @@ -1069,6 +1083,8 @@ "label.guest.netmask": "Guest netmask", "label.guest.networks": "Guest Networks", "label.guest.os": "Guest OS", +"label.guest.os.category": "Guest OS Category", +"label.guest.os.categories": "Guest OS Categories", "label.guest.os.hypervisor.mappings": "Guest OS mappings", "label.guest.start.ip": "Guest start IP", "label.guest.traffic": "Guest traffic", @@ -1107,7 +1123,9 @@ "label.host": "IP address", "label.host.alerts": "Hosts in alert state", "label.host.name": "Host name", +"label.host.ovftool.version": "OVFTool Version", "label.host.tag": "Host tag", +"label.host.virtv2v.version": "Virt-v2v Version", "label.hostcontrolstate": "Compute Resource Status", "label.hostid": "Host", "label.hostname": "Host", @@ -1143,6 +1161,8 @@ "label.ikelifetime": "IKE lifetime (second)", "label.ikepolicy": "IKE policy", "label.ikeversion": "IKE version", +"label.image": "Image", +"label.image.type": "Image type", "label.images": "Images", "label.imagestoreid": "Secondary Storage", "label.import.backup.offering": "Import backup offering", @@ -1294,11 +1314,13 @@ "label.kubernetes": "Kubernetes", "label.kubernetes.access.details": "The kubernetes nodes can be accessed via ssh using:
ssh -i [ssh_key] -p [port_number] cloud@[public_ip_address]

where,
ssh_key: points to the ssh private key file corresponding to the key that was associated while creating the Kubernetes cluster. If no ssh key was provided during Kubernetes cluster creation, use the ssh private key of the management server.
port_number: can be obtained from the Port Forwarding Tab (Public Port column)", "label.kubernetes.cluster": "Kubernetes cluster", +"label.kubernetes.cluster.add.nodes.to.cluster": "Add nodes to Kubernetes cluster", "label.kubernetes.cluster.create": "Create Kubernetes cluster", "label.kubernetes.cluster.delete": "Delete Kubernetes cluster", "label.kubernetes.cluster.scale": "Scale Kubernetes cluster", "label.kubernetes.cluster.start": "Start Kubernetes cluster", "label.kubernetes.cluster.stop": "Stop Kubernetes cluster", +"label.kubernetes.cluster.remove.nodes.from.cluster": "Remove nodes from Kubernetes cluster", "label.kubernetes.cluster.upgrade": "Upgrade Kubernetes cluster", "label.kubernetes.dashboard": "Kubernetes dashboard UI", "label.kubernetes.dashboard.create.token": "Create token for Kubernetes dashboard", @@ -1507,6 +1529,7 @@ "label.monitor.url": "URL Path", "label.monthly": "Monthly", "label.more.access.dashboard.ui": "More about accessing dashboard UI", +"label.mount.cks.iso.on.vr": "Use CKS packages from Virtual Router", "label.mount.sharedfs": "Mount Shared FileSystem via NFS", "label.move.down.row": "Move down one row", "label.move.to.bottom": "Move to bottom", @@ -1590,6 +1613,7 @@ "label.no.items": "No available Items", "label.no.matching.offering": "No matching offering found", "label.no.matching.network": "No matching Networks found", +"label.node.version": "Node version", "label.no.usage.records": "No usage records found", "label.noderootdisksize": "Node root disk size (in GB)", "label.nodiskcache": "No disk cache", @@ -1651,6 +1675,7 @@ "label.operator.equal": "Equals to", "label.optional": "Optional", "label.order": "Order", +"label.os": "Operating System", "label.oscategoryid": "OS category", "label.oscategoryname": "OS category name", "label.osname": "OS name", @@ -1806,6 +1831,7 @@ "label.provisioningtype.fat": "Fat provisioning", "label.provisioningtype.sparse": "Sparse provisioning", "label.provisioningtype.thin": "Thin provisioning", +"label.public": "Public", "label.publicmtu": "Public Interface MTU", "label.public.interface": "Public interface", "label.public.ip": "Public IP address", @@ -1876,6 +1902,7 @@ "label.read.io": "Read (IO)", "label.readonly": "Read-Only", "label.reason": "Reason", +"label.rebalance": "Rebalance", "label.reboot": "Reboot", "label.recent.deliveries": "Recent deliveries", "label.receivedbytes": "Bytes received", @@ -1892,7 +1919,8 @@ "label.region": "Region", "label.register.oauth": "Register OAuth", "label.register.template": "Register Template", -"label.register.user.data": "Register a userdata", +"label.register.user.data": "Register User Data", +"label.register.cni.config": "Register CNI Configuration", "label.reinstall.vm": "Reinstall Instance", "label.reject": "Reject", "label.related": "Related", @@ -1910,6 +1938,7 @@ "label.remove": "Remove", "label.remove.annotation": "Remove comment", "label.remove.bgp.peer": "Remove BGP peer", +"label.remove.cni.configuration": "Remove CNI configuration", "label.remove.egress.rule": "Remove egress rule", "label.remove.interface.route.table": "Remove Tungsten interface route table", "label.remove.ip.range": "Remove IP range", @@ -1919,6 +1948,7 @@ "label.remove.logical.router": "Remove logical router", "label.remove.network.offering": "Remove Network offering", "label.remove.network.route.table": "Remove Tungsten Fabric Network routing table", +"label.remove.nodes": "Remove nodes from Kubernetes cluster", "label.remove.pf": "Remove port forwarding rule", "label.remove.policy": "Remove policy", "label.remove.project.account": "Remove Account from project", @@ -2097,6 +2127,9 @@ "label.service.lb.netscaler.servicepackages": "Netscaler service packages", "label.service.lb.netscaler.servicepackages.description": "Service package description", "label.service.offering": "Service offering", +"label.service.offering.controlnodes": "Compute offering for Control Nodes", +"label.service.offering.etcdnodes": "Compute offering for etcd Nodes", +"label.service.offering.workernodes": "Compute offering for Worker Nodes", "label.service.staticnat.associatepublicip": "Associate public IP", "label.service.staticnat.elasticipcheckbox": "Elastic IP", "label.servicegroupuuid": "Service Group", @@ -2305,7 +2338,8 @@ "label.tariffvalue": "Tariff value", "label.tcp": "TCP", "label.tcp.proxy": "TCP proxy", -"label.template": "Select a template", +"label.template": "Template", +"label.template.select": "Select a template", "label.templatetag": "Tag", "label.template.select.existing": "Select an existing template", "label.template.temporary.import": "Use a temporary template for import", @@ -2467,6 +2501,7 @@ "label.usagetypedescription": "Usage description", "label.use.kubectl.access.cluster": "kubectl and kubeconfig file to access cluster", "label.use.local.timezone": "Use local timezone", +"label.use.router.ip.resolver": "Use Virtual Router IP as resolver", "label.used": "Used", "label.usehttps": "Use HTTPS", "label.usenewdiskoffering": "Replace disk offering?", @@ -2579,6 +2614,8 @@ "label.vnmc": "VNMC", "label.volgroup": "Volume group", "label.volume": "Volume", +"label.vms.empty": "No VMs available to be added to the Kubernetes cluster", +"label.vms.remove.empty": "No external VMs present in the Kubernetes cluster to be removed", "label.volume.empty": "No data volumes attached to this Instance", "label.volume.encryption.support": "Volume Encryption Supported", "label.volume.metrics": "Volume Metrics", @@ -2668,11 +2705,15 @@ "label.bucket.delete": "Delete Bucket", "label.quotagib": "Quota in GiB", "label.encryption": "Encryption", +"label.etcdnodes": "Number of etcd nodes", "label.versioning": "Versioning", "label.objectlocking": "Object Lock", "label.bucket.policy": "Bucket Policy", "label.usersecretkey": "Secret Key", "label.create.bucket": "Create Bucket", +"label.cniconfiguration": "CNI Configuration", +"label.cniconfigname": "Associated CNI Configuration", +"label.cniconfigparams": "CNI Configuration parameters", "label.lease.enable": "Enable Lease", "label.lease.enable.tooltip": "The Instance Lease feature allows to set a lease duration (in days) for instances, after which they automatically expire. Upon expiry, the instance can either be stopped (powered off) or destroyed, based on the configured policy", "label.instance.lease": "Instance lease", @@ -2698,6 +2739,7 @@ "message.action.delete.ingress.rule": "Please confirm that you want to delete this ingress rule.", "message.action.delete.ipv4.subnet": "Please confirm that you want to delete this IPv4 subnet.", "message.action.delete.guest.os": "Please confirm that you want to delete this guest os. System defined entry cannot be deleted.", +"message.action.delete.guest.os.category": "Please confirm that you want to delete this guest os category.", "message.action.delete.guest.os.hypervisor.mapping": "Please confirm that you want to delete this guest os hypervisor mapping. System defined entry cannot be deleted.", "message.action.delete.instance.group": "Please confirm that you want to delete the Instance group.", "message.action.delete.interface.static.route": "Please confirm that you want to remove this interface Static Route?", @@ -2859,6 +2901,8 @@ "message.adding.host": "Adding host", "message.adding.netscaler.device": "Adding Netscaler device", "message.adding.netscaler.provider": "Adding Netscaler provider", +"message.adding.nodes.to.cluster": "Adding nodes to Kubernetes cluster", +"message.removing.nodes.from.cluster": "Removing nodes from Kubernetes cluster", "message.advanced.security.group": "Choose this if you wish to use security groups to provide guest Instance isolation.", "message.allowed": "Allowed", "message.alert.show.all.stats.data": "This may return a lot of data depending on VM statistics and retention settings", @@ -3059,6 +3103,7 @@ "message.desc.reset.ssh.key.pair": "Please specify a ssh key pair that you would like to add to this Instance.", "message.desc.secondary.storage": "Each zone must have at least one NFS or secondary storage server. We will add the first one now. Secondary storage stores Instance Templates, ISO images, and Instance disk volume Snapshots. This server must be available to all hosts in the zone.

Provide the IP address and exported path.", "message.desc.register.user.data": "Please fill in the following data to register a User data.", +"message.desc.register.cni.config": "Please fill in the following data to register CNI Configuration as user data.", "message.desc.registered.user.data": "Registered a User Data.", "message.desc.zone": "A zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. A zone consists of one or more pods (each of which contains hosts and primary storage servers) and a secondary storage server which is shared by all pods in the zone.", "message.desc.zone.edge": "A zone is the largest organizational unit in CloudStack, and it typically corresponds to a single datacenter. Zones provide physical isolation and redundancy. An edge zone consists of one or more hosts (each of which provides local storage as primary storage servers). Only shared and L2 Networks can be deployed in such zones and functionalities that require secondary storages are not supported.", @@ -3306,6 +3351,7 @@ "message.installwizard.tooltip.tungsten.provider.name": "Tungsten provider name is required", "message.installwizard.tooltip.tungsten.provider.port": "Tungsten provider port is required", "message.installwizard.tooltip.tungsten.provider.vrouterport": "Tungsten provider vrouter port is required", +"message.instance.architecture": "Please select Instance architecture", "message.instances.managed": "Instances controlled by CloudStack.", "message.instances.unmanaged": "Instances not controlled by CloudStack.", "message.instances.migrate.vmware": "Instances that can be migrated from VMware.", @@ -3316,10 +3362,12 @@ "message.iso.arch": "Please select an ISO architecture", "message.iso.desc": "Disc image containing data or bootable media for OS.", "message.kubeconfig.cluster.not.available": "Kubernetes cluster kubeconfig not available currently.", +"message.kubernetes.cluster.add.nodes": "Please confirm that you want to add the following nodes to the cluster", "message.kubernetes.cluster.delete": "Please confirm that you want to destroy the cluster.", "message.kubernetes.cluster.scale": "Please select desired cluster configuration.", "message.kubernetes.cluster.start": "Please confirm that you want to start the cluster.", "message.kubernetes.cluster.stop": "Please confirm that you want to stop the cluster.", +"message.kubernetes.cluster.remove.nodes": "Please confirm that you want to remove the following nodes from the cluster", "message.kubernetes.cluster.upgrade": "Please select new Kubernetes version.", "message.kubernetes.version.delete": "Please confirm that you want to delete this Kubernetes version.", "message.l2.network.unsupported.for.nsx": "L2 networks aren't supported for NSX enabled zones", @@ -3397,6 +3445,7 @@ "message.password.reset.success": "Password has been reset successfully. Please login using your new credentials.", "message.path": "Path : ", "message.path.description": "NFS: exported path from the server. VMFS: /datacenter name/datastore name. SharedMountPoint: path where primary storage is mounted, such as /mnt/primary.", +"message.please.confirm.remove.cni.configuration": "Please confirm that you want to remove this CNI Configuration", "message.please.confirm.remove.ssh.key.pair": "Please confirm that you want to remove this SSH key pair.", "message.please.confirm.remove.user.data": "Please confirm that you want to remove this Userdata", "message.please.enter.valid.value": "Please enter a valid value.", @@ -3523,6 +3572,8 @@ "message.success.add.network.acl": "Successfully added Network ACL list", "message.success.add.network.static.route": "Successfully added Network Static Route", "message.success.add.network.permissions": "Successfully added Network permissions", +"message.success.add.nodes.to.cluster": "Successfully added nodes to Kubernetes cluster", +"message.success.remove.nodes.from.cluster": "Successfully removed nodes from Kubernetes cluster", "message.success.add.physical.network": "Successfully added Physical Network", "message.success.add.object.storage": "Successfully added Object Storage", "message.success.add.policy.rule": "Successfully added Policy rule", diff --git a/ui/src/api/index.js b/ui/src/api/index.js index 7ab87780a9d..85c46c483b2 100644 --- a/ui/src/api/index.js +++ b/ui/src/api/index.js @@ -23,18 +23,10 @@ import { ACCESS_TOKEN } from '@/store/mutation-types' -export function api (command, args = {}, method = 'GET', data = {}) { - let params = {} +export function getAPI (command, args = {}) { args.command = command args.response = 'json' - if (data) { - params = new URLSearchParams() - Object.entries(data).forEach(([key, value]) => { - params.append(key, value) - }) - } - const sessionkey = vueProps.$localStorage.get(ACCESS_TOKEN) || Cookies.get('sessionkey') if (sessionkey) { args.sessionkey = sessionkey @@ -45,8 +37,30 @@ export function api (command, args = {}, method = 'GET', data = {}) { ...args }, url: '/', - method, - data: params || {} + method: 'GET' + }) +} + +export function postAPI (command, data = {}) { + const params = new URLSearchParams() + params.append('command', command) + params.append('response', 'json') + if (data) { + Object.entries(data).forEach(([key, value]) => { + if (value !== undefined && value !== null && value !== '') { + params.append(key, value) + } + }) + } + + const sessionkey = vueProps.$localStorage.get(ACCESS_TOKEN) || Cookies.get('sessionkey') + if (sessionkey) { + params.append('sessionkey', sessionkey) + } + return axios({ + url: '/', + method: 'POST', + data: params }) } @@ -56,7 +70,7 @@ export function login (arg) { } // Logout before login is called to purge any duplicate sessionkey cookies - api('logout') + postAPI('logout') const params = new URLSearchParams() params.append('command', 'login') @@ -66,7 +80,7 @@ export function login (arg) { params.append('response', 'json') return axios({ url: '/', - method: 'post', + method: 'POST', data: params, headers: { 'content-type': 'application/x-www-form-urlencoded' @@ -77,7 +91,7 @@ export function login (arg) { export function logout () { message.destroy() notification.destroy() - return api('logout') + return postAPI('logout') } export function oauthlogin (arg) { @@ -86,7 +100,7 @@ export function oauthlogin (arg) { } // Logout before login is called to purge any duplicate sessionkey cookies - api('logout') + postAPI('logout') const params = new URLSearchParams() params.append('command', 'oauthlogin') diff --git a/ui/src/components/header/AnnouncementBanner.vue b/ui/src/components/header/AnnouncementBanner.vue new file mode 100644 index 00000000000..af4a760c110 --- /dev/null +++ b/ui/src/components/header/AnnouncementBanner.vue @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/components/header/SamlDomainSwitcher.vue b/ui/src/components/header/SamlDomainSwitcher.vue index 082bab7bf13..e0799bef4d8 100644 --- a/ui/src/components/header/SamlDomainSwitcher.vue +++ b/ui/src/components/header/SamlDomainSwitcher.vue @@ -51,7 +51,7 @@ diff --git a/ui/src/components/view/ImageStoreSelectView.vue b/ui/src/components/view/ImageStoreSelectView.vue index 13c5a68e5b1..7871703cd8e 100644 --- a/ui/src/components/view/ImageStoreSelectView.vue +++ b/ui/src/components/view/ImageStoreSelectView.vue @@ -75,7 +75,7 @@ + + diff --git a/ui/src/components/widgets/Console.vue b/ui/src/components/widgets/Console.vue index 6c16c7546a7..e348a25bb21 100644 --- a/ui/src/components/widgets/Console.vue +++ b/ui/src/components/widgets/Console.vue @@ -28,7 +28,7 @@ + diff --git a/ui/src/views/compute/KubernetesRemoveNodes.vue b/ui/src/views/compute/KubernetesRemoveNodes.vue new file mode 100644 index 00000000000..57d6409b784 --- /dev/null +++ b/ui/src/views/compute/KubernetesRemoveNodes.vue @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + diff --git a/ui/src/views/compute/KubernetesServiceTab.vue b/ui/src/views/compute/KubernetesServiceTab.vue index 1c2c83a35ce..fc8f9c60213 100644 --- a/ui/src/views/compute/KubernetesServiceTab.vue +++ b/ui/src/views/compute/KubernetesServiceTab.vue @@ -117,9 +117,15 @@
{{ cksSshPortSharedNetwork }}
-
+
{{ cksSshStartingPort + index }}
+
+ {{ parseInt(etcdSshPort) + parseInt(getEtcdIndex(record.name)) - 1 }} +
+ + + + + + diff --git a/ui/src/views/compute/wizard/MultiDiskSelection.vue b/ui/src/views/compute/wizard/MultiDiskSelection.vue index 8344508ad33..f9ae73860e6 100644 --- a/ui/src/views/compute/wizard/MultiDiskSelection.vue +++ b/ui/src/views/compute/wizard/MultiDiskSelection.vue @@ -72,7 +72,7 @@ + + diff --git a/ui/src/views/compute/wizard/OsBasedImageSelection.vue b/ui/src/views/compute/wizard/OsBasedImageSelection.vue new file mode 100644 index 00000000000..57fb4b355ac --- /dev/null +++ b/ui/src/views/compute/wizard/OsBasedImageSelection.vue @@ -0,0 +1,373 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/wizard/OsBasedImageSelectionSearchView.vue b/ui/src/views/compute/wizard/OsBasedImageSelectionSearchView.vue new file mode 100644 index 00000000000..95088d19b5a --- /dev/null +++ b/ui/src/views/compute/wizard/OsBasedImageSelectionSearchView.vue @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/views/compute/wizard/OwnershipSelection.vue b/ui/src/views/compute/wizard/OwnershipSelection.vue index a2c7ea4c1f8..59f781a75aa 100644 --- a/ui/src/views/compute/wizard/OwnershipSelection.vue +++ b/ui/src/views/compute/wizard/OwnershipSelection.vue @@ -128,7 +128,7 @@ diff --git a/ui/src/views/dashboard/CapacityDashboard.vue b/ui/src/views/dashboard/CapacityDashboard.vue index 53a3d87aa23..7e0b8180ac8 100644 --- a/ui/src/views/dashboard/CapacityDashboard.vue +++ b/ui/src/views/dashboard/CapacityDashboard.vue @@ -332,7 +332,7 @@