diff --git a/.asf.yaml b/.asf.yaml index 43f0351bc7c..0603bf8e487 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -54,10 +54,12 @@ github: - gpordeus - hsato03 - bernardodemarco - - abh1sar - FelipeM525 - lucas-a-martins - nicoschmdt + - abh1sar + - sudo87 + - rosi-shapeblue protected_branches: ~ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ceffb42c79b..0d101777c70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,8 @@ jobs: component/test_cpu_limits component/test_cpu_max_limits component/test_cpu_project_limits - component/test_deploy_vm_userdata_multi_nic", + component/test_deploy_vm_userdata_multi_nic + component/test_deploy_vm_lease", "component/test_egress_fw_rules component/test_invalid_gw_nm component/test_ip_reservation", @@ -236,7 +237,7 @@ jobs: - name: Install Python dependencies run: | - python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid + python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose - name: Install jacoco dependencies run: | diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 8248e48022a..dccd7174e54 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -54,7 +54,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} restore-keys: | ${{ runner.os }}-m2 diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index c36bceb2b90..d31f55980a8 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -56,7 +56,7 @@ jobs: uses: actions/cache@v4 with: path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} restore-keys: | ${{ runner.os }}-m2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6d006938f3..17ff4badc97 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,9 +4,9 @@ Contributing to Apache CloudStack (ACS) Summary ------- This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions. -These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added. +These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project, and you will submit a Pull Request for your changes to be added. -_Lets get started!!!_ +_Let's get started!!!_ Bug fixes --------- @@ -26,7 +26,7 @@ No back porting / cherry-picking features to existing branches! PendingReleaseNotes file ------------------------ -When developing a new feature or making a (major) change to a existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can +When developing a new feature or making a (major) change to an existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can use this file as a source of information when compiling the Release Notes for a new release. When adding information to the PendingReleaseNotes file make sure that you write a good and understandable description of the new feature or change which you have developed. @@ -38,9 +38,9 @@ Fork the code In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack) -Fork the repository by clicking on the 'Fork' button on the top right hand side. The fork will happen and you will be taken to your own fork of the repository. Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'. You will paste this URL when doing the following `git clone` command. +Fork the repository by clicking on the 'Fork' button on the top right hand side. The fork will happen, and you will be taken to your own fork of the repository. Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'. You will paste this URL when doing the following `git clone` command. -On your computer, follow these steps to setup a local repository for working on ACS: +On your computer, follow these steps to set up a local repository for working on ACS: ```bash $ git clone https://github.com/YOUR_ACCOUNT/cloudstack.git @@ -92,9 +92,9 @@ $ git rebase main Make a GitHub Pull Request to contribute your changes ----------------------------------------------------- -When you are happy with your changes and you are ready to contribute them, you will create a Pull Request on GitHub to do so. This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub. +When you are happy with your changes, and you are ready to contribute them, you will create a Pull Request on GitHub to do so. This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub. -Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. Incase of UI PRs, a screenshot is preferred. +Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. In case of UI PRs, a screenshot is preferred. > **IMPORTANT:** Make sure you have rebased your `feature_x` branch to include the latest code from `upstream/main` _before_ you do this. diff --git a/INSTALL.md b/INSTALL.md index e133e7d7b91..5f04f2d5599 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -20,10 +20,10 @@ Install tools and dependencies used for development: Set up Maven (3.6.0): - # wget http://www.us.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz - # tar -zxvf apache-maven-3.6.3-bin.tar.gz -C /usr/local + # wget https://dlcdn.apache.org/maven/maven-3/3.9.9/binaries/apache-maven-3.9.9-bin.tar.gz + # tar -zxvf apache-maven-3.9.9-bin.tar.gz -C /usr/local # cd /usr/local - # ln -s apache-maven-3.6.3 maven + # ln -s apache-maven-3.9.9 maven # echo export M2_HOME=/usr/local/maven >> ~/.bashrc # or .zshrc or .profile # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile # source ~/.bashrc @@ -37,6 +37,7 @@ Setup up NodeJS (LTS): Start the MySQL service: $ service mysqld start + $ mysql_secure_installation ### Using jenv and/or pyenv for Version Management @@ -86,13 +87,33 @@ Start the management server: If this works, you've successfully setup a single server Apache CloudStack installation. -Open the following URL on your browser to access the Management Server UI: - - http://localhost:8080/client/ +To access the Management Server UI, follow the following procedure: The default credentials are; user: admin, password: password and the domain field should be left blank which is defaulted to the ROOT domain. +## To bring up CloudStack UI + +Move to UI Directory + + $ cd /path/to/cloudstack/ui + +To install dependencies. + + $ npm install + +To build the project. + + $ npm build + +For Development Mode. + + $ npm start + +Make sure to set CS_URL=http://localhost:8080/client on .env.local file on ui. + +You should be able to run the management server on http://localhost:5050 + ## Building with non-redistributable plugins CloudStack supports several plugins that depend on libraries with distribution restrictions. diff --git a/agent/bindir/cloud-setup-agent.in b/agent/bindir/cloud-setup-agent.in index 18de64089ed..9927c22eebc 100755 --- a/agent/bindir/cloud-setup-agent.in +++ b/agent/bindir/cloud-setup-agent.in @@ -20,6 +20,19 @@ import os import logging import sys import socket + +# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ---- +# ---- We do this so cloud_utils can be looked up in the following order: +# ---- 1) Sources directory +# ---- 2) waf configured PYTHONDIR +# ---- 3) System Python path +for pythonpath in ( + "@PYTHONDIR@", + os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"), + ): + if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath) +# ---- End snippet of code ---- + from cloudutils.cloudException import CloudRuntimeException, CloudInternalException from cloudutils.utilities import initLoging, bash from cloudutils.configFileOps import configFileOps diff --git a/agent/bindir/libvirtqemuhook.in b/agent/bindir/libvirtqemuhook.in index e17944d8353..4cc6ed7a1d2 100755 --- a/agent/bindir/libvirtqemuhook.in +++ b/agent/bindir/libvirtqemuhook.in @@ -20,6 +20,19 @@ import sys import os import subprocess from threading import Timer + +# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ---- +# ---- We do this so cloud_utils can be looked up in the following order: +# ---- 1) Sources directory +# ---- 2) waf configured PYTHONDIR +# ---- 3) System Python path +for pythonpath in ( + "@PYTHONDIR@", + os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"), + ): + if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath) +# ---- End snippet of code ---- + from xml.dom.minidom import parse from cloudutils.configFileOps import configFileOps from cloudutils.networkConfig import networkConfig diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index bff7078fd9f..e70acee229d 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -441,3 +441,9 @@ iscsi.session.cleanup.enabled=false # Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used #backoff.seconds= + +# Timeout (in seconds) to wait for the snapshot reversion to complete. +# revert.snapshot.timeout=10800 + +# Timeout (in seconds) to wait for the incremental snapshot to complete. +# incremental.snapshot.timeout=10800 diff --git a/agent/src/main/java/com/cloud/agent/Agent.java b/agent/src/main/java/com/cloud/agent/Agent.java index 0a76bfbb4f8..fcd4234a136 100644 --- a/agent/src/main/java/com/cloud/agent/Agent.java +++ b/agent/src/main/java/com/cloud/agent/Agent.java @@ -342,7 +342,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater logger.info("Attempted to connect to the server, but received an unexpected exception, trying again...", e); } } - shell.updateConnectedHost(); + shell.updateConnectedHost(((NioClient)connection).getHost()); scavengeOldAgentObjects(); } @@ -617,15 +617,11 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } protected void reconnect(final Link link) { - reconnect(link, null, null, false); + reconnect(link, null, false); } - protected void reconnect(final Link link, String preferredHost, List avoidHostList, boolean forTransfer) { + protected void reconnect(final Link link, String preferredMSHost, boolean forTransfer) { if (!(forTransfer || reconnectAllowed)) { - return; - } - - if (!reconnectAllowed) { logger.debug("Reconnect requested but it is not allowed {}", () -> getLinkLog(link)); return; } @@ -637,19 +633,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater serverResource.disconnected(); logger.info("Lost connection to host: {}. Attempting reconnection while we still have {} commands in progress.", shell.getConnectedHost(), commandsInProgress.get()); stopAndCleanupConnection(true); + String host = preferredMSHost; + if (org.apache.commons.lang3.StringUtils.isBlank(host)) { + host = shell.getNextHost(); + } + List avoidMSHostList = shell.getAvoidHosts(); do { - final String host = shell.getNextHost(); - connection = new NioClient(getAgentName(), host, shell.getPort(), shell.getWorkers(), shell.getSslHandshakeTimeout(), this); - logger.info("Reconnecting to host: {}", host); - try { - connection.start(); - } catch (final NioConnectionException e) { - logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e); - stopAndCleanupConnection(false); + if (CollectionUtils.isEmpty(avoidMSHostList) || !avoidMSHostList.contains(host)) { + connection = new NioClient(getAgentName(), host, shell.getPort(), shell.getWorkers(), shell.getSslHandshakeTimeout(), this); + logger.info("Reconnecting to host: {}", host); + try { + connection.start(); + } catch (final NioConnectionException e) { + logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e); + stopAndCleanupConnection(false); + } } shell.getBackoffAlgorithm().waitBeforeRetry(); + host = shell.getNextHost(); } while (!connection.isStartup()); - shell.updateConnectedHost(); + shell.updateConnectedHost(((NioClient)connection).getHost()); logger.info("Connected to the host: {}", shell.getConnectedHost()); } @@ -797,6 +800,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } commandsInProgress.incrementAndGet(); try { + if (cmd.isReconcile()) { + cmd.setRequestSequence(request.getSequence()); + } answer = serverResource.executeRequest(cmd); } finally { commandsInProgress.decrementAndGet(); @@ -922,7 +928,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater return new SetupCertificateAnswer(true); } - private void processManagementServerList(final List msList, final String lbAlgorithm, final Long lbCheckInterval) { + private void processManagementServerList(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) { if (CollectionUtils.isNotEmpty(msList) && StringUtils.isNotEmpty(lbAlgorithm)) { try { final String newMSHosts = String.format("%s%s%s", com.cloud.utils.StringUtils.toCSVList(msList), IAgentShell.hostLbAlgorithmSeparator, lbAlgorithm); @@ -934,6 +940,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater throw new CloudRuntimeException("Could not persist received management servers list", e); } } + shell.setAvoidHosts(avoidMsList); if ("shuffle".equals(lbAlgorithm)) { scheduleHostLBCheckerTask(0); } else { @@ -942,16 +949,18 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } private Answer setupManagementServerList(final SetupMSListCommand cmd) { - processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); + processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); return new SetupMSListAnswer(true); } private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) { try { if (CollectionUtils.isNotEmpty(cmd.getMsList())) { - processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); + processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval()); } - migrateAgentConnection(cmd.getAvoidMsList()); + Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job")).schedule(() -> { + migrateAgentConnection(cmd.getAvoidMsList()); + }, 3, TimeUnit.SECONDS); } catch (Exception e) { String errMsg = "Migrate agent connection failed, due to " + e.getMessage(); logger.debug(errMsg, e); @@ -972,25 +981,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater throw new CloudRuntimeException("No other Management Server hosts to migrate"); } - String preferredHost = null; + String preferredMSHost = null; for (String msHost : msHostsList) { try (final Socket socket = new Socket()) { socket.connect(new InetSocketAddress(msHost, shell.getPort()), 5000); - preferredHost = msHost; + preferredMSHost = msHost; break; } catch (final IOException e) { throw new CloudRuntimeException("Management server host: " + msHost + " is not reachable, to migrate connection"); } } - if (preferredHost == null) { + if (preferredMSHost == null) { throw new CloudRuntimeException("Management server host(s) are not reachable, to migrate connection"); } - logger.debug("Management server host " + preferredHost + " is found to be reachable, trying to reconnect"); + logger.debug("Management server host " + preferredMSHost + " is found to be reachable, trying to reconnect"); shell.resetHostCounter(); + shell.setAvoidHosts(avoidMsList); shell.setConnectionTransfer(true); - reconnect(link, preferredHost, avoidMsList, true); + reconnect(link, preferredMSHost, true); } public void processResponse(final Response response, final Link link) { @@ -1003,14 +1013,23 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater for (final IAgentControlListener listener : controlListeners) { listener.processControlResponse(response, (AgentControlAnswer)answer); } - } else if (answer instanceof PingAnswer && (((PingAnswer) answer).isSendStartup()) && reconnectAllowed) { - logger.info("Management server requested startup command to reinitialize the agent"); - sendStartup(link); + } else if (answer instanceof PingAnswer) { + processPingAnswer((PingAnswer) answer); } else { updateLastPingResponseTime(); } } + private void processPingAnswer(final PingAnswer answer) { + if ((answer.isSendStartup()) && reconnectAllowed) { + logger.info("Management server requested startup command to reinitialize the agent"); + sendStartup(link); + } else { + serverResource.processPingAnswer((PingAnswer) answer); + } + shell.setAvoidHosts(answer.getAvoidMsList()); + } + public void processReadyCommand(final Command cmd) { final ReadyCommand ready = (ReadyCommand)cmd; // Set human readable sizes; @@ -1027,7 +1046,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater } verifyAgentArch(ready.getArch()); - processManagementServerList(ready.getMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval()); + processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval()); logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName()); } @@ -1073,6 +1092,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater Answer answer = null; commandsInProgress.incrementAndGet(); try { + if (command.isReconcile()) { + command.setRequestSequence(req.getSequence()); + } answer = serverResource.executeRequest(command); } finally { commandsInProgress.decrementAndGet(); @@ -1374,26 +1396,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater if (msList == null || msList.length < 1) { return; } - final String preferredHost = msList[0]; + final String preferredMSHost = msList[0]; final String connectedHost = shell.getConnectedHost(); logger.debug("Running preferred host checker task, connected host={}, preferred host={}", - connectedHost, preferredHost); - if (preferredHost == null || preferredHost.equals(connectedHost) || link == null) { + connectedHost, preferredMSHost); + if (preferredMSHost == null || preferredMSHost.equals(connectedHost) || link == null) { return; } boolean isHostUp = false; try (final Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(preferredHost, shell.getPort()), 5000); + socket.connect(new InetSocketAddress(preferredMSHost, shell.getPort()), 5000); isHostUp = true; } catch (final IOException e) { - logger.debug("Host: {} is not reachable", preferredHost); + logger.debug("Host: {} is not reachable", preferredMSHost); } if (isHostUp && link != null && commandsInProgress.get() == 0) { if (logger.isDebugEnabled()) { - logger.debug("Preferred host {} is found to be reachable, trying to reconnect", preferredHost); + logger.debug("Preferred host {} is found to be reachable, trying to reconnect", preferredMSHost); } shell.resetHostCounter(); - reconnect(link); + reconnect(link, preferredMSHost, false); } } catch (Throwable t) { logger.error("Error caught while attempting to connect to preferred host", t); diff --git a/agent/src/main/java/com/cloud/agent/AgentShell.java b/agent/src/main/java/com/cloud/agent/AgentShell.java index aea7fd3a8de..4862e7e001e 100644 --- a/agent/src/main/java/com/cloud/agent/AgentShell.java +++ b/agent/src/main/java/com/cloud/agent/AgentShell.java @@ -66,6 +66,7 @@ public class AgentShell implements IAgentShell, Daemon { private String _zone; private String _pod; private String _host; + private List _avoidHosts; private String _privateIp; private int _port; private int _proxyPort; @@ -76,7 +77,6 @@ public class AgentShell implements IAgentShell, Daemon { private volatile boolean _exit = false; private int _pingRetries; private final List _agents = new ArrayList(); - private String hostToConnect; private String connectedHost; private Long preferredHostCheckInterval; private boolean connectionTransfer = false; @@ -121,7 +121,7 @@ public class AgentShell implements IAgentShell, Daemon { if (_hostCounter >= hosts.length) { _hostCounter = 0; } - hostToConnect = hosts[_hostCounter % hosts.length]; + String hostToConnect = hosts[_hostCounter % hosts.length]; _hostCounter++; return hostToConnect; } @@ -143,11 +143,10 @@ public class AgentShell implements IAgentShell, Daemon { } @Override - public void updateConnectedHost() { - connectedHost = hostToConnect; + public void updateConnectedHost(String connectedHost) { + this.connectedHost = connectedHost; } - @Override public void resetHostCounter() { _hostCounter = 0; @@ -166,6 +165,16 @@ public class AgentShell implements IAgentShell, Daemon { } } + @Override + public void setAvoidHosts(List avoidHosts) { + _avoidHosts = avoidHosts; + } + + @Override + public List getAvoidHosts() { + return _avoidHosts; + } + @Override public String getPrivateIp() { return _privateIp; diff --git a/agent/src/main/java/com/cloud/agent/IAgentShell.java b/agent/src/main/java/com/cloud/agent/IAgentShell.java index c0ecd90ae69..9eefa6d2eee 100644 --- a/agent/src/main/java/com/cloud/agent/IAgentShell.java +++ b/agent/src/main/java/com/cloud/agent/IAgentShell.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.agent; +import java.util.List; import java.util.Map; import java.util.Properties; @@ -63,9 +64,13 @@ public interface IAgentShell { String[] getHosts(); + void setAvoidHosts(List hosts); + + List getAvoidHosts(); + long getLbCheckerInterval(Long receivedLbInterval); - void updateConnectedHost(); + void updateConnectedHost(String connectedHost); String getConnectedHost(); diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 61cd27fff77..69537621673 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -816,7 +816,17 @@ public class AgentProperties{ * Data type: Integer.
* Default value: null */ - public static final Property SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", null, Integer.class); + public static final Property SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", 30, Integer.class); + + /** + * Timeout (in seconds) to wait for the incremental snapshot to complete. + * */ + public static final Property INCREMENTAL_SNAPSHOT_TIMEOUT = new Property<>("incremental.snapshot.timeout", 10800); + + /** + * Timeout (in seconds) to wait for the snapshot reversion to complete. + * */ + public static final Property REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800); public static class Property { private String name; diff --git a/agent/src/test/java/com/cloud/agent/AgentShellTest.java b/agent/src/test/java/com/cloud/agent/AgentShellTest.java index 6d9758cc3dc..d8def24a603 100644 --- a/agent/src/test/java/com/cloud/agent/AgentShellTest.java +++ b/agent/src/test/java/com/cloud/agent/AgentShellTest.java @@ -358,7 +358,7 @@ public class AgentShellTest { AgentShell shell = new AgentShell(); shell.setHosts("test"); shell.getNextHost(); - shell.updateConnectedHost(); + shell.updateConnectedHost("test"); Assert.assertEquals(expected, shell.getConnectedHost()); } diff --git a/api/src/main/java/com/cloud/agent/api/Command.java b/api/src/main/java/com/cloud/agent/api/Command.java index eb979c0060b..4766c30ead2 100644 --- a/api/src/main/java/com/cloud/agent/api/Command.java +++ b/api/src/main/java/com/cloud/agent/api/Command.java @@ -35,6 +35,23 @@ public abstract class Command { Continue, Stop } + public enum State { + CREATED, // Command is created by management server + STARTED, // Command is started by agent + PROCESSING, // Processing by agent + PROCESSING_IN_BACKEND, // Processing in backend by agent + COMPLETED, // Operation succeeds by agent or management server + FAILED, // Operation fails by agent + RECONCILE_RETRY, // Ready for retry of reconciliation + RECONCILING, // Being reconciled by management server + RECONCILED, // Reconciled by management server + RECONCILE_SKIPPED, // Skip the reconciliation as the resource state is inconsistent with the command + RECONCILE_FAILED, // Fail to reconcile by management server + TIMED_OUT, // Timed out on management server or agent + INTERRUPTED, // Interrupted by management server or agent (for example agent is restarted), + DANGLED_IN_BACKEND // Backend process which cannot be processed normally (for example agent is restarted) + } + public static final String HYPERVISOR_TYPE = "hypervisorType"; // allow command to carry over hypervisor or other environment related context info @@ -42,6 +59,7 @@ public abstract class Command { protected Map contextMap = new HashMap(); private int wait; //in second private boolean bypassHostMaintenance = false; + private transient long requestSequence = 0L; protected Command() { this.wait = 0; @@ -82,6 +100,10 @@ public abstract class Command { return contextMap.get(name); } + public Map getContextMap() { + return contextMap; + } + public boolean allowCaching() { return true; } @@ -94,6 +116,18 @@ public abstract class Command { this.bypassHostMaintenance = bypassHostMaintenance; } + public boolean isReconcile() { + return false; + } + + public long getRequestSequence() { + return requestSequence; + } + + public void setRequestSequence(long requestSequence) { + this.requestSequence = requestSequence; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/api/src/main/java/com/cloud/agent/api/to/DiskTO.java b/api/src/main/java/com/cloud/agent/api/to/DiskTO.java index d22df2df172..5664de79091 100644 --- a/api/src/main/java/com/cloud/agent/api/to/DiskTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/DiskTO.java @@ -46,7 +46,7 @@ public class DiskTO { private Long diskSeq; private String path; private Volume.Type type; - private Map _details; + private Map details; public DiskTO() { @@ -92,10 +92,10 @@ public class DiskTO { } public void setDetails(Map details) { - _details = details; + this.details = details; } public Map getDetails() { - return _details; + return details; } } diff --git a/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java b/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java index bd08ce81101..d65ec0e3daa 100644 --- a/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/NetworkTO.java @@ -36,7 +36,7 @@ public class NetworkTO { protected TrafficType type; protected URI broadcastUri; protected URI isolationUri; - protected boolean isSecurityGroupEnabled; + protected boolean securityGroupEnabled; protected String name; protected String ip6address; protected String ip6gateway; @@ -112,7 +112,7 @@ public class NetworkTO { } public void setSecurityGroupEnabled(boolean enabled) { - this.isSecurityGroupEnabled = enabled; + this.securityGroupEnabled = enabled; } /** @@ -221,7 +221,7 @@ public class NetworkTO { } public boolean isSecurityGroupEnabled() { - return this.isSecurityGroupEnabled; + return this.securityGroupEnabled; } public void setIp6Dns1(String ip6Dns1) { diff --git a/api/src/main/java/com/cloud/agent/api/to/NicTO.java b/api/src/main/java/com/cloud/agent/api/to/NicTO.java index 573363c04fb..ca95fcfd679 100644 --- a/api/src/main/java/com/cloud/agent/api/to/NicTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/NicTO.java @@ -86,6 +86,14 @@ public class NicTO extends NetworkTO { this.nicUuid = uuid; } + public String getNicUuid() { + return nicUuid; + } + + public void setNicUuid(String nicUuid) { + this.nicUuid = nicUuid; + } + @Override public String toString() { return new StringBuilder("[Nic:").append(type).append("-").append(ip).append("-").append(broadcastUri).append("]").toString(); diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index 6f24b1cd6ca..6c415ec1df1 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -61,7 +61,7 @@ public class VirtualMachineTO { @LogLevel(LogLevel.Log4jLevel.Off) String vncPassword; String vncAddr; - Map params; + Map details; String uuid; String bootType; String bootMode; @@ -191,7 +191,11 @@ public class VirtualMachineTO { return maxSpeed; } - public boolean getLimitCpuUse() { + public boolean isEnableHA() { + return enableHA; + } + + public boolean isLimitCpuUse() { return limitCpuUse; } @@ -289,11 +293,11 @@ public class VirtualMachineTO { } public Map getDetails() { - return params; + return details; } public void setDetails(Map params) { - this.params = params; + this.details = params; } public String getUuid() { diff --git a/api/src/main/java/com/cloud/configuration/ConfigurationService.java b/api/src/main/java/com/cloud/configuration/ConfigurationService.java index 97d4b42974b..13a44ef05b0 100644 --- a/api/src/main/java/com/cloud/configuration/ConfigurationService.java +++ b/api/src/main/java/com/cloud/configuration/ConfigurationService.java @@ -201,11 +201,12 @@ public interface ConfigurationService { * TODO * @param allocationState * TODO + * @param storageAccessGroups * @return the new pod if successful, null otherwise * @throws * @throws */ - Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState); + Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState, List storageAccessGroups); /** * Creates a mutual exclusive IP range in the pod with same gateway, netmask. diff --git a/api/src/main/java/com/cloud/cpu/CPU.java b/api/src/main/java/com/cloud/cpu/CPU.java index 4e1b9f5a501..3016e542db6 100644 --- a/api/src/main/java/com/cloud/cpu/CPU.java +++ b/api/src/main/java/com/cloud/cpu/CPU.java @@ -16,52 +16,55 @@ // under the License. package com.cloud.cpu; -import com.cloud.utils.exception.CloudRuntimeException; import org.apache.commons.lang3.StringUtils; -import java.util.LinkedHashMap; -import java.util.Map; - public class CPU { + public enum CPUArch { + x86("i686", 32), + amd64("x86_64", 64), + arm64("aarch64", 64); - public static final String archX86Identifier = "i686"; - public static final String archX86_64Identifier = "x86_64"; - public static final String archARM64Identifier = "aarch64"; + private final String type; + private final int bits; - public static class CPUArch { - private static final Map cpuArchMap = new LinkedHashMap<>(); - - public static final CPUArch archX86 = new CPUArch(archX86Identifier, 32); - public static final CPUArch amd64 = new CPUArch(archX86_64Identifier, 64); - public static final CPUArch arm64 = new CPUArch(archARM64Identifier, 64); - - private String type; - private int bits; - - public CPUArch(String type, int bits) { + CPUArch(String type, int bits) { this.type = type; this.bits = bits; - cpuArchMap.put(type, this); + } + + public static CPUArch getDefault() { + return amd64; } public String getType() { - return this.type; + return type; } public int getBits() { - return this.bits; + return bits; } public static CPUArch fromType(String type) { if (StringUtils.isBlank(type)) { - return amd64; + return getDefault(); } - switch (type) { - case archX86Identifier: return archX86; - case archX86_64Identifier: return amd64; - case archARM64Identifier: return arm64; - default: throw new CloudRuntimeException(String.format("Unsupported arch type: %s", type)); + for (CPUArch arch : values()) { + if (arch.type.equals(type)) { + return arch; + } } + throw new IllegalArgumentException("Unsupported arch type: " + type); + } + + public static String getTypesAsCSV() { + StringBuilder sb = new StringBuilder(); + for (CPUArch arch : values()) { + sb.append(arch.getType()).append(","); + } + if (sb.length() > 0) { + sb.setLength(sb.length() - 1); + } + return sb.toString(); } } } diff --git a/api/src/main/java/com/cloud/dc/Pod.java b/api/src/main/java/com/cloud/dc/Pod.java index 1cbab36f3bd..17c5b615d4b 100644 --- a/api/src/main/java/com/cloud/dc/Pod.java +++ b/api/src/main/java/com/cloud/dc/Pod.java @@ -43,4 +43,6 @@ public interface Pod extends InfrastructureEntity, Grouping, Identity, InternalI AllocationState getAllocationState(); boolean getExternalDhcp(); + + String getStorageAccessGroups(); } diff --git a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java index 2697311d2b9..d127e4bdd66 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java @@ -62,7 +62,7 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner { "vm.allocation.algorithm", "Advanced", "random", - "Order in which hosts within a cluster will be considered for VM/volume allocation. The value can be 'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', or 'firstfitleastconsumed'.", + "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', or 'firstfitleastconsumed'.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 862a6e21fa8..e194085a9d9 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -465,6 +465,7 @@ public class EventTypes { public static final String EVENT_ENABLE_PRIMARY_STORAGE = "ENABLE.PS"; public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS"; public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL"; + public static final String EVENT_CONFIGURE_STORAGE_ACCESS = "CONFIGURE.STORAGE.ACCESS"; public static final String EVENT_CHANGE_STORAGE_POOL_SCOPE = "CHANGE.STORAGE.POOL.SCOPE"; // VPN @@ -739,6 +740,13 @@ public class EventTypes { //Purge resources public static final String EVENT_PURGE_EXPUNGED_RESOURCES = "PURGE.EXPUNGED.RESOURCES"; + // Management Server + public static final String EVENT_MS_MAINTENANCE_PREPARE = "MS.MAINTENANCE.PREPARE"; + public static final String EVENT_MS_MAINTENANCE_CANCEL = "MS.MAINTENANCE.CANCEL"; + public static final String EVENT_MS_SHUTDOWN_PREPARE = "MS.SHUTDOWN.PREPARE"; + public static final String EVENT_MS_SHUTDOWN_CANCEL = "MS.SHUTDOWN.CANCEL"; + public static final String EVENT_MS_SHUTDOWN = "MS.SHUTDOWN"; + // OBJECT STORE public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE"; public static final String EVENT_OBJECT_STORE_DELETE = "OBJECT.STORE.DELETE"; @@ -788,6 +796,11 @@ public class EventTypes { // Resource Limit public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE"; + public static final String VM_LEASE_EXPIRED = "VM.LEASE.EXPIRED"; + public static final String VM_LEASE_DISABLED = "VM.LEASE.DISABLED"; + public static final String VM_LEASE_CANCELLED = "VM.LEASE.CANCELLED"; + public static final String VM_LEASE_EXPIRING = "VM.LEASE.EXPIRING"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking @@ -1233,6 +1246,12 @@ public class EventTypes { entityEventDetails.put(EVENT_UPDATE_IMAGE_STORE_ACCESS_STATE, ImageStore.class); entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); + entityEventDetails.put(EVENT_MS_MAINTENANCE_PREPARE, "ManagementServer"); + entityEventDetails.put(EVENT_MS_MAINTENANCE_CANCEL, "ManagementServer"); + entityEventDetails.put(EVENT_MS_SHUTDOWN_PREPARE, "ManagementServer"); + entityEventDetails.put(EVENT_MS_SHUTDOWN_CANCEL, "ManagementServer"); + entityEventDetails.put(EVENT_MS_SHUTDOWN, "ManagementServer"); + //Object Store entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class); entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class); @@ -1276,6 +1295,12 @@ public class EventTypes { entityEventDetails.put(EVENT_SHAREDFS_DESTROY, SharedFS.class); entityEventDetails.put(EVENT_SHAREDFS_EXPUNGE, SharedFS.class); entityEventDetails.put(EVENT_SHAREDFS_RECOVER, SharedFS.class); + + // VM Lease + entityEventDetails.put(VM_LEASE_EXPIRED, VirtualMachine.class); + entityEventDetails.put(VM_LEASE_EXPIRING, VirtualMachine.class); + entityEventDetails.put(VM_LEASE_DISABLED, VirtualMachine.class); + entityEventDetails.put(VM_LEASE_CANCELLED, VirtualMachine.class); } public static boolean isNetworkEvent(String eventType) { diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index afac6df5631..8b9aa4ed791 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -213,4 +213,6 @@ public interface Host extends StateObject, Identity, Partition, HAResour ResourceState getResourceState(); CPU.CPUArch getArch(); + + String getStorageAccessGroups(); } diff --git a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java index a13c1b3a6a8..84e898528b5 100644 --- a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java +++ b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java @@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster; import org.apache.cloudstack.acl.ControlledEntity; +import com.cloud.user.Account; import com.cloud.uservm.UserVm; import com.cloud.utils.component.Adapter; @@ -26,4 +27,5 @@ public interface KubernetesServiceHelper extends Adapter { ControlledEntity findByUuid(String uuid); ControlledEntity findByVmId(long vmId); void checkVmCanBeDestroyed(UserVm userVm); + void cleanupForAccount(Account account); } diff --git a/api/src/main/java/com/cloud/org/Cluster.java b/api/src/main/java/com/cloud/org/Cluster.java index 5124168084c..b0aa6bb04cf 100644 --- a/api/src/main/java/com/cloud/org/Cluster.java +++ b/api/src/main/java/com/cloud/org/Cluster.java @@ -41,4 +41,6 @@ public interface Cluster extends Grouping, Partition { ManagedState getManagedState(); CPU.CPUArch getArch(); + + String getStorageAccessGroups(); } diff --git a/api/src/main/java/com/cloud/resource/ResourceService.java b/api/src/main/java/com/cloud/resource/ResourceService.java index 562c3c418df..3cdf8fc64e9 100644 --- a/api/src/main/java/com/cloud/resource/ResourceService.java +++ b/api/src/main/java/com/cloud/resource/ResourceService.java @@ -95,4 +95,11 @@ public interface ResourceService { boolean releaseHostReservation(Long hostId); + void updatePodStorageAccessGroups(long podId, List newStorageAccessGroups); + + void updateZoneStorageAccessGroups(long zoneId, List newStorageAccessGroups); + + void updateClusterStorageAccessGroups(Long clusterId, List newStorageAccessGroups); + + void updateHostStorageAccessGroups(Long hostId, List newStorageAccessGroups); } diff --git a/api/src/main/java/com/cloud/storage/MigrationOptions.java b/api/src/main/java/com/cloud/storage/MigrationOptions.java index a39a2a7c827..8b642d09e29 100644 --- a/api/src/main/java/com/cloud/storage/MigrationOptions.java +++ b/api/src/main/java/com/cloud/storage/MigrationOptions.java @@ -24,6 +24,7 @@ public class MigrationOptions implements Serializable { private String srcPoolUuid; private Storage.StoragePoolType srcPoolType; + private Long srcPoolClusterId; private Type type; private ScopeType scopeType; private String srcBackingFilePath; @@ -38,21 +39,23 @@ public class MigrationOptions implements Serializable { public MigrationOptions() { } - public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcBackingFilePath, boolean copySrcTemplate, ScopeType scopeType) { + public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcBackingFilePath, boolean copySrcTemplate, ScopeType scopeType, Long srcPoolClusterId) { this.srcPoolUuid = srcPoolUuid; this.srcPoolType = srcPoolType; this.type = Type.LinkedClone; this.scopeType = scopeType; this.srcBackingFilePath = srcBackingFilePath; this.copySrcTemplate = copySrcTemplate; + this.srcPoolClusterId = srcPoolClusterId; } - public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcVolumeUuid, ScopeType scopeType) { + public MigrationOptions(String srcPoolUuid, Storage.StoragePoolType srcPoolType, String srcVolumeUuid, ScopeType scopeType, Long srcPoolClusterId) { this.srcPoolUuid = srcPoolUuid; this.srcPoolType = srcPoolType; this.type = Type.FullClone; this.scopeType = scopeType; this.srcVolumeUuid = srcVolumeUuid; + this.srcPoolClusterId = srcPoolClusterId; } public String getSrcPoolUuid() { @@ -63,6 +66,10 @@ public class MigrationOptions implements Serializable { return srcPoolType; } + public Long getSrcPoolClusterId() { + return srcPoolClusterId; + } + public ScopeType getScopeType() { return scopeType; } public String getSrcBackingFilePath() { diff --git a/api/src/main/java/com/cloud/storage/Snapshot.java b/api/src/main/java/com/cloud/storage/Snapshot.java index fc919e442b2..c0a7b812ed9 100644 --- a/api/src/main/java/com/cloud/storage/Snapshot.java +++ b/api/src/main/java/com/cloud/storage/Snapshot.java @@ -48,7 +48,7 @@ public interface Snapshot extends ControlledEntity, Identity, InternalIdentity, } public enum State { - Allocated, Creating, CreatedOnPrimary, BackingUp, BackedUp, Copying, Destroying, Destroyed, + Allocated, Creating, CreatedOnPrimary, BackingUp, BackedUp, Copying, Destroying, Destroyed, Hidden, //it's a state, user can't see the snapshot from ui, while the snapshot may still exist on the storage Error; diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index b8df75cd3e4..6f7b62911b6 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.ChangeStoragePoolScopeCmd; +import org.apache.cloudstack.api.command.admin.storage.ConfigureStorageAccessCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; @@ -99,6 +100,8 @@ public interface StorageService { StoragePool disablePrimaryStoragePool(Long id); + boolean configureStorageAccess(ConfigureStorageAccessCmd cmd); + StoragePool getStoragePool(long id); boolean deleteImageStore(DeleteImageStoreCmd cmd); diff --git a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java index f43d5331222..db702a61f2b 100644 --- a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java +++ b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java @@ -17,6 +17,7 @@ package com.cloud.storage; import java.util.Date; +import java.util.List; import org.apache.cloudstack.api.InternalIdentity; @@ -25,6 +26,8 @@ public interface VMTemplateStorageResourceAssoc extends InternalIdentity { UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED } + List PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS); + String getInstallPath(); long getTemplateId(); diff --git a/api/src/main/java/com/cloud/user/Account.java b/api/src/main/java/com/cloud/user/Account.java index 6be4d0a48f6..1cc5eae64a6 100644 --- a/api/src/main/java/com/cloud/user/Account.java +++ b/api/src/main/java/com/cloud/user/Account.java @@ -71,6 +71,7 @@ public interface Account extends ControlledEntity, InternalIdentity, Identity { } public static final long ACCOUNT_ID_SYSTEM = 1; + public static final long ACCOUNT_ID_ADMIN = 2; public String getAccountName(); diff --git a/api/src/main/java/com/cloud/vm/VmDetailConstants.java b/api/src/main/java/com/cloud/vm/VmDetailConstants.java index 29803d5271b..ef6ec27f172 100644 --- a/api/src/main/java/com/cloud/vm/VmDetailConstants.java +++ b/api/src/main/java/com/cloud/vm/VmDetailConstants.java @@ -101,4 +101,17 @@ public interface VmDetailConstants { String VMWARE_HOST_NAME = String.format("%s-host", VMWARE_TO_KVM_PREFIX); String VMWARE_DISK = String.format("%s-disk", VMWARE_TO_KVM_PREFIX); String VMWARE_MAC_ADDRESSES = String.format("%s-mac-addresses", VMWARE_TO_KVM_PREFIX); + + // TPM + String VIRTUAL_TPM_ENABLED = "virtual.tpm.enabled"; + String VIRTUAL_TPM_MODEL = "virtual.tpm.model"; + String VIRTUAL_TPM_VERSION = "virtual.tpm.version"; + + // CPU mode and model, ADMIN only + String GUEST_CPU_MODE = "guest.cpu.mode"; + String GUEST_CPU_MODEL = "guest.cpu.model"; + + String INSTANCE_LEASE_EXPIRY_DATE = "leaseexpirydate"; + String INSTANCE_LEASE_EXPIRY_ACTION = "leaseexpiryaction"; + String INSTANCE_LEASE_EXECUTION = "leaseactionexecution"; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3e8b329cac7..304e43009f2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -269,7 +269,10 @@ public class ApiConstants { public static final String INTERNAL_DNS2 = "internaldns2"; public static final String INTERNET_PROTOCOL = "internetprotocol"; public static final String INTERVAL_TYPE = "intervaltype"; - public static final String LOCATION_TYPE = "locationtype"; + public static final String INSTANCE_LEASE_DURATION = "leaseduration"; + public static final String INSTANCE_LEASE_ENABLED = "instanceleaseenabled"; + public static final String INSTANCE_LEASE_EXPIRY_ACTION = "leaseexpiryaction"; + public static final String INSTANCE_LEASE_EXPIRY_DATE= "leaseexpirydate"; public static final String IOPS_READ_RATE = "iopsreadrate"; public static final String IOPS_READ_RATE_MAX = "iopsreadratemax"; public static final String IOPS_READ_RATE_MAX_LENGTH = "iopsreadratemaxlength"; @@ -317,11 +320,13 @@ public class ApiConstants { public static final String LAST_BOOT = "lastboottime"; public static final String LAST_SERVER_START = "lastserverstart"; public static final String LAST_SERVER_STOP = "lastserverstop"; + public static final String LEASED = "leased"; public static final String LEVEL = "level"; public static final String LENGTH = "length"; public static final String LIMIT = "limit"; public static final String LIMIT_CPU_USE = "limitcpuuse"; public static final String LIST_HOSTS = "listhosts"; + public static final String LOCATION_TYPE = "locationtype"; public static final String LOCK = "lock"; public static final String LUN = "lun"; public static final String LBID = "lbruleid"; @@ -456,6 +461,7 @@ public class ApiConstants { public static final String SENT = "sent"; public static final String SENT_BYTES = "sentbytes"; public static final String SERIAL = "serial"; + public static final String SERVICE_IP = "serviceip"; public static final String SERVICE_OFFERING_ID = "serviceofferingid"; public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; @@ -485,18 +491,22 @@ public class ApiConstants { public static final String STATE = "state"; public static final String STATS = "stats"; public static final String STATUS = "status"; + public static final String STORAGE_TYPE = "storagetype"; + public static final String STORAGE_POLICY = "storagepolicy"; + public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; public static final String STORAGE_CAPABILITIES = "storagecapabilities"; public static final String STORAGE_CUSTOM_STATS = "storagecustomstats"; - public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled"; - public static final String STORAGE_POLICY = "storagepolicy"; - public static final String STORAGE_POOL = "storagepool"; - public static final String STORAGE_TYPE = "storagetype"; public static final String SUBNET = "subnet"; public static final String OWNER = "owner"; public static final String SWAP_OWNER = "swapowner"; public static final String SYSTEM_VM_TYPE = "systemvmtype"; public static final String TAGS = "tags"; public static final String STORAGE_TAGS = "storagetags"; + public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups"; + public static final String STORAGE_ACCESS_GROUP = "storageaccessgroup"; + public static final String CLUSTER_STORAGE_ACCESS_GROUPS = "clusterstorageaccessgroups"; + public static final String POD_STORAGE_ACCESS_GROUPS = "podstorageaccessgroups"; + public static final String ZONE_STORAGE_ACCESS_GROUPS = "zonestorageaccessgroups"; public static final String SUCCESS = "success"; public static final String SUITABLE_FOR_VM = "suitableforvirtualmachine"; public static final String SUPPORTS_STORAGE_SNAPSHOT = "supportsstoragesnapshot"; @@ -1150,6 +1160,7 @@ public class ApiConstants { public static final String PENDING_JOBS_COUNT = "pendingjobscount"; public static final String AGENTS_COUNT = "agentscount"; public static final String AGENTS = "agents"; + public static final String LAST_AGENTS = "lastagents"; public static final String PUBLIC_MTU = "publicmtu"; public static final String PRIVATE_MTU = "privatemtu"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java b/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java index d4fdeddc9a9..03dc37325d4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiErrorCode.java @@ -30,6 +30,7 @@ public enum ApiErrorCode { UNSUPPORTED_ACTION_ERROR(432), API_LIMIT_EXCEED(429), + SERVICE_UNAVAILABLE(503), INTERNAL_ERROR(530), ACCOUNT_ERROR(531), ACCOUNT_RESOURCE_LIMIT_ERROR(532), diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index ea0d946ee41..e2d132c2ae6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -310,6 +310,8 @@ public interface ResponseGenerator { PodResponse createPodResponse(Pod pod, Boolean showCapacities); + PodResponse createMinimalPodResponse(Pod pod); + ZoneResponse createZoneResponse(ResponseView view, DataCenter dataCenter, Boolean showCapacities, Boolean showResourceIcon); DataCenterGuestIpv6PrefixResponse createDataCenterGuestIpv6PrefixResponse(DataCenterGuestIpv6Prefix prefix); @@ -324,6 +326,8 @@ public interface ResponseGenerator { ClusterResponse createClusterResponse(Cluster cluster, Boolean showCapacities); + ClusterResponse createMinimalClusterResponse(Cluster cluster); + FirewallRuleResponse createPortForwardingRuleResponse(PortForwardingRule fwRule); IpForwardingRuleResponse createIpForwardingRuleResponse(StaticNatRule fwRule); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java index 69cb43ce40e..15265f561e7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/AddClusterCmd.java @@ -118,6 +118,12 @@ public class AddClusterCmd extends BaseCmd { private String ovm3cluster; @Parameter(name = ApiConstants.OVM3_VIP, type = CommandType.STRING, required = false, description = "Ovm3 vip to use for pool (and cluster)") private String ovm3vip; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, + type = CommandType.LIST, collectionType = CommandType.STRING, + description = "comma separated list of storage access groups for the hosts in the cluster", + since = "4.21.0") + private List storageAccessGroups; + public String getOvm3Pool() { return ovm3pool; } @@ -192,6 +198,10 @@ public class AddClusterCmd extends BaseCmd { this.clusterType = type; } + public List getStorageAccessGroups() { + return storageAccessGroups; + } + @Override public long getEntityOwnerId() { return Account.ACCOUNT_ID_SYSTEM; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java index 67d0678410c..9cc39503fbf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/ListClustersCmd.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.api.command.admin.cluster; import java.util.ArrayList; import java.util.List; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; @@ -28,7 +27,9 @@ import org.apache.cloudstack.api.response.ClusterResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.commons.lang3.StringUtils; +import com.cloud.cpu.CPU; import com.cloud.org.Cluster; import com.cloud.utils.Pair; @@ -68,6 +69,16 @@ public class ListClustersCmd extends BaseListCmd { @Parameter(name = ApiConstants.SHOW_CAPACITIES, type = CommandType.BOOLEAN, description = "flag to display the capacity of the clusters") private Boolean showCapacities; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "CPU arch of the clusters", + since = "4.20.1") + private String arch; + + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUP, type = CommandType.STRING, + description = "the name of the storage access group", + since = "4.21.0") + private String storageAccessGroup; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -112,6 +123,22 @@ public class ListClustersCmd extends BaseListCmd { return showCapacities; } + public CPU.CPUArch getArch() { + return StringUtils.isBlank(arch) ? null : CPU.CPUArch.fromType(arch); + } + + public String getStorageAccessGroup() { + return storageAccessGroup; + } + + public ListClustersCmd() { + + } + + public ListClustersCmd(String storageAccessGroup) { + this.storageAccessGroup = storageAccessGroup; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/UpdateClusterCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/UpdateClusterCmd.java index c4ee87380ed..816285e3430 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/UpdateClusterCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/cluster/UpdateClusterCmd.java @@ -130,7 +130,7 @@ public class UpdateClusterCmd extends BaseCmd { } Cluster result = _resourceService.updateCluster(this); if (result != null) { - ClusterResponse clusterResponse = _responseGenerator.createClusterResponse(cluster, false); + ClusterResponse clusterResponse = _responseGenerator.createClusterResponse(result, false); clusterResponse.setResponseName(getCommandName()); this.setResponseObject(clusterResponse); } else { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddHostCmd.java index ca27837aa88..6531444b52e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/AddHostCmd.java @@ -75,6 +75,12 @@ public class AddHostCmd extends BaseCmd { @Parameter(name = ApiConstants.HOST_TAGS, type = CommandType.LIST, collectionType = CommandType.STRING, description = "list of tags to be added to the host") private List hostTags; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, + type = CommandType.LIST, collectionType = CommandType.STRING, + description = "comma separated list of storage access groups for the host", + since = "4.21.0") + private List storageAccessGroups; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -115,6 +121,10 @@ public class AddHostCmd extends BaseCmd { return hostTags; } + public List getStorageAccessGroups() { + return storageAccessGroups; + } + public String getAllocationState() { return allocationState; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index 5e229521efe..a71150a69e2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -21,7 +21,6 @@ import java.util.EnumSet; import java.util.List; import java.util.Map; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -35,7 +34,9 @@ import org.apache.cloudstack.api.response.ManagementServerResponse; import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.commons.lang3.StringUtils; +import com.cloud.cpu.CPU; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; @@ -109,6 +110,14 @@ public class ListHostsCmd extends BaseListCmd { @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "the id of the management server", since="4.21.0") private Long managementServerId; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, description = "CPU Arch of the host", since = "4.20.1") + private String arch; + + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUP, type = CommandType.STRING, + description = "the name of the storage access group", + since = "4.21.0") + private String storageAccessGroup; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -197,6 +206,22 @@ public class ListHostsCmd extends BaseListCmd { return managementServerId; } + public CPU.CPUArch getArch() { + return StringUtils.isBlank(arch) ? null : CPU.CPUArch.fromType(arch); + } + + public String getStorageAccessGroup() { + return storageAccessGroup; + } + + public ListHostsCmd() { + + } + + public ListHostsCmd(String storageAccessGroup) { + this.storageAccessGroup = storageAccessGroup; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java index 8f6d5413d72..4cadaad0e47 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmd.java @@ -34,7 +34,9 @@ import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.VsphereStoragePoliciesResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.api.response.DiskOfferingResponse; +import org.apache.cloudstack.vm.lease.VMLeaseManager; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections.CollectionUtils; @@ -251,7 +253,15 @@ public class CreateServiceOfferingCmd extends BaseCmd { since="4.20") private Boolean purgeResources; + @Parameter(name = ApiConstants.INSTANCE_LEASE_DURATION, + type = CommandType.INTEGER, + description = "Number of days instance is leased for.", + since = "4.21.0") + private Integer leaseDuration; + @Parameter(name = ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION, type = CommandType.STRING, since = "4.21.0", + description = "Lease expiry action, valid values are STOP and DESTROY") + private String leaseExpiryAction; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -487,6 +497,22 @@ public class CreateServiceOfferingCmd extends BaseCmd { return false; } + public VMLeaseManager.ExpiryAction getLeaseExpiryAction() { + if (StringUtils.isBlank(leaseExpiryAction)) { + return null; + } + VMLeaseManager.ExpiryAction action = EnumUtils.getEnumIgnoreCase(VMLeaseManager.ExpiryAction.class, leaseExpiryAction); + if (action == null) { + throw new InvalidParameterValueException("Invalid value configured for leaseexpiryaction, valid values are: " + + com.cloud.utils.EnumUtils.listValues(VMLeaseManager.ExpiryAction.values())); + } + return action; + } + + public Integer getLeaseDuration() { + return leaseDuration; + } + public boolean isPurgeResources() { return Boolean.TRUE.equals(purgeResources); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/CreatePodCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/CreatePodCmd.java index c1d9a6db429..36ad00e6e4a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/CreatePodCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/CreatePodCmd.java @@ -30,6 +30,8 @@ import org.apache.cloudstack.api.response.ZoneResponse; import com.cloud.dc.Pod; import com.cloud.user.Account; +import java.util.List; + @APICommand(name = "createPod", description = "Creates a new Pod.", responseObject = PodResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreatePodCmd extends BaseCmd { @@ -63,6 +65,12 @@ public class CreatePodCmd extends BaseCmd { @Parameter(name = ApiConstants.ALLOCATION_STATE, type = CommandType.STRING, description = "Allocation state of this Pod for allocation of new resources") private String allocationState; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, + type = CommandType.LIST, collectionType = CommandType.STRING, + description = "comma separated list of storage access groups for the hosts in the pod", + since = "4.21.0") + private List storageAccessGroups; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -95,6 +103,10 @@ public class CreatePodCmd extends BaseCmd { return allocationState; } + public List getStorageAccessGroups() { + return storageAccessGroups; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -111,7 +123,7 @@ public class CreatePodCmd extends BaseCmd { @Override public void execute() { - Pod result = _configService.createPod(getZoneId(), getPodName(), getStartIp(), getEndIp(), getGateway(), getNetmask(), getAllocationState()); + Pod result = _configService.createPod(getZoneId(), getPodName(), getStartIp(), getEndIp(), getGateway(), getNetmask(), getAllocationState(), getStorageAccessGroups()); if (result != null) { PodResponse response = _responseGenerator.createPodResponse(result, false); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java index 5ad0b457ced..ca5635d4fe4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/pod/ListPodsByCmd.java @@ -55,6 +55,11 @@ public class ListPodsByCmd extends BaseListCmd { @Parameter(name = ApiConstants.SHOW_CAPACITIES, type = CommandType.BOOLEAN, description = "flag to display the capacity of the pods") private Boolean showCapacities; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUP, type = CommandType.STRING, + description = "the name of the storage access group", + since = "4.21.0") + private String storageAccessGroup; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -79,6 +84,18 @@ public class ListPodsByCmd extends BaseListCmd { return showCapacities; } + public String getStorageAccessGroup() { + return storageAccessGroup; + } + + public ListPodsByCmd() { + + } + + public ListPodsByCmd(String storageAccessGroup) { + this.storageAccessGroup = storageAccessGroup; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java index e0cdc0dcf80..9e34c05ce21 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/router/ListRoutersCmd.java @@ -16,8 +16,6 @@ // under the License. package org.apache.cloudstack.api.command.admin.router; -import org.apache.commons.lang.BooleanUtils; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -32,7 +30,10 @@ import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VpcResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import com.cloud.cpu.CPU; import com.cloud.network.router.VirtualRouter.Role; import com.cloud.vm.VirtualMachine; @@ -86,6 +87,11 @@ public class ListRoutersCmd extends BaseListProjectAndAccountResourcesCmd { description = "if true is passed for this parameter, also fetch last executed health check results for the router. Default is false") private Boolean fetchHealthCheckResults; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "CPU arch of the router", + since = "4.20.1") + private String arch; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -146,6 +152,10 @@ public class ListRoutersCmd extends BaseListProjectAndAccountResourcesCmd { return BooleanUtils.isTrue(fetchHealthCheckResults); } + public CPU.CPUArch getArch() { + return StringUtils.isBlank(arch) ? null : CPU.CPUArch.fromType(arch); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ConfigureStorageAccessCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ConfigureStorageAccessCmd.java new file mode 100644 index 00000000000..bfa2589921f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ConfigureStorageAccessCmd.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import java.util.List; + +import com.cloud.event.EventTypes; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.response.ClusterResponse; +import org.apache.cloudstack.api.response.HostResponse; +import org.apache.cloudstack.api.response.PodResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.api.response.ZoneResponse; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; + +import com.cloud.user.Account; + +@APICommand(name = "configureStorageAccess", description = "Configure the storage access groups on zone/pod/cluster/host and storage, accordingly connections to the storage pools", responseObject = SuccessResponse.class, since = "4.21.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ConfigureStorageAccessCmd extends BaseAsyncCmd { + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "UUID of the zone") + private Long zoneId; + + @Parameter(name = ApiConstants.POD_ID, type = CommandType.UUID, entityType = PodResponse.class, description = "UUID of the pod") + private Long podId; + + @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, description = "UUID of the cluster") + private Long clusterId; + + @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "UUID of the host") + private Long hostId; + + @Parameter(name = ApiConstants.STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, description = "UUID of the Storage Pool") + private Long storageId; + + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, type = CommandType.LIST, collectionType = CommandType.STRING, + description = "comma separated list of storage access groups for connecting the storage pools and the hosts", + since = "4.21.0") + private List storageAccessGroups; + + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getZoneId() { + return zoneId; + } + + public Long getPodId() { + return podId; + } + + public Long getClusterId() { + return clusterId; + } + + public Long getHostId() { + return hostId; + } + + public Long getStorageId() { + return storageId; + } + + public List getStorageAccessGroups() { + return storageAccessGroups; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.StoragePool; + } + + @Override + public void execute() { + try { + boolean result = _storageService.configureStorageAccess(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to configure storage access"); + } + } catch (Exception e) { + logger.debug("Failed to configure storage access ", e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to configure storage access, " + e.getMessage()); + } + } + + @Override + public String getEventType() { + return EventTypes.EVENT_CONFIGURE_STORAGE_ACCESS; + } + + @Override + public String getEventDescription() { + return "configuring storage access groups"; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java index 75813a7aabf..cbe4b8c06b3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/CreateStoragePoolCmd.java @@ -61,6 +61,10 @@ public class CreateStoragePoolCmd extends BaseCmd { @Parameter(name = ApiConstants.TAGS, type = CommandType.STRING, description = "the tags for the storage pool") private String tags; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, type = CommandType.STRING, + description = "comma separated list of storage access groups for connecting to hosts having those specific groups", since = "4.21.0") + private String storageAccessGroups; + @Parameter(name = ApiConstants.URL, type = CommandType.STRING, required = true, description = "the URL of the storage pool") private String url; @@ -115,6 +119,10 @@ public class CreateStoragePoolCmd extends BaseCmd { return tags; } + public String getStorageAccessGroups() { + return storageAccessGroups; + } + public String getUrl() { return url; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStorageAccessGroupsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStorageAccessGroupsCmd.java new file mode 100644 index 00000000000..d2a1757839f --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStorageAccessGroupsCmd.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.StorageAccessGroupResponse; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.response.ListResponse; + +@APICommand(name = "listStorageAccessGroups", description = "Lists storage access groups", responseObject = StorageAccessGroupResponse.class, since = "4.21.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ListStorageAccessGroupsCmd extends BaseListCmd { + + // /////////////////////////////////////////////////// + // ////////////// API parameters ///////////////////// + // /////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.NAME, type = BaseCmd.CommandType.STRING, description = "Name of the Storage access group") + private String name; + + // /////////////////////////////////////////////////// + // ///////////////// Accessors /////////////////////// + // /////////////////////////////////////////////////// + + public String getName() { + return name; + } + + // /////////////////////////////////////////////////// + // ///////////// API Implementation/////////////////// + // /////////////////////////////////////////////////// + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.StoragePool; + } + + @Override + public void execute() { + ListResponse response = _queryService.searchForStorageAccessGroups(this); + + response.setResponseName(getCommandName()); + + setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java index 57a87939b6b..0f2c9f3416e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolsCmd.java @@ -41,7 +41,7 @@ public class ListStoragePoolsCmd extends BaseListCmd { @Parameter(name = ApiConstants.CLUSTER_ID, type = CommandType.UUID, entityType = ClusterResponse.class, - description = "list storage pools belongig to the specific cluster") + description = "list storage pools belonging to the specific cluster") private Long clusterId; @Parameter(name = ApiConstants.IP_ADDRESS, type = CommandType.STRING, description = "the IP address for the storage pool") @@ -74,6 +74,10 @@ public class ListStoragePoolsCmd extends BaseListCmd { @Parameter(name = ApiConstants.STORAGE_CUSTOM_STATS, type = CommandType.BOOLEAN, description = "If true, lists the custom stats of the storage pool", since = "4.18.1") private Boolean customStats; + + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUP, type = CommandType.STRING, description = "the name of the storage access group", since = "4.21.0") + private String storageAccessGroup; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -134,6 +138,17 @@ public class ListStoragePoolsCmd extends BaseListCmd { return customStats != null && customStats; } + public String getStorageAccessGroup() { + return storageAccessGroup; + } + + public ListStoragePoolsCmd() { + } + + public ListStoragePoolsCmd(String storageAccessGroup) { + this.storageAccessGroup = storageAccessGroup; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ListSystemVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ListSystemVMsCmd.java index e8e5ee0ebad..13113e17ea4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ListSystemVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/systemvm/ListSystemVMsCmd.java @@ -19,7 +19,6 @@ package org.apache.cloudstack.api.command.admin.systemvm; import java.util.ArrayList; import java.util.List; - import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -31,7 +30,9 @@ import org.apache.cloudstack.api.response.PodResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.SystemVmResponse; import org.apache.cloudstack.api.response.ZoneResponse; +import org.apache.commons.lang3.StringUtils; +import com.cloud.cpu.CPU; import com.cloud.utils.Pair; import com.cloud.vm.VirtualMachine; @@ -74,6 +75,11 @@ public class ListSystemVMsCmd extends BaseListCmd { since = "3.0.1") private Long storageId; + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "CPU arch of the system VM", + since = "4.20.1") + private String arch; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -110,6 +116,10 @@ public class ListSystemVMsCmd extends BaseListCmd { return storageId; } + public CPU.CPUArch getArch() { + return StringUtils.isBlank(arch) ? null : CPU.CPUArch.fromType(arch); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/zone/CreateZoneCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/zone/CreateZoneCmd.java index 24660e41ed9..f3c1bab260b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/zone/CreateZoneCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/zone/CreateZoneCmd.java @@ -31,6 +31,8 @@ import org.apache.cloudstack.context.CallContext; import com.cloud.dc.DataCenter; import com.cloud.user.Account; +import java.util.List; + @APICommand(name = "createZone", description = "Creates a Zone.", responseObject = ZoneResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateZoneCmd extends BaseCmd { @@ -88,6 +90,11 @@ public class CreateZoneCmd extends BaseCmd { @Parameter(name = ApiConstants.IS_EDGE, type = CommandType.BOOLEAN, description = "true if the zone is an edge zone, false otherwise", since = "4.18.0") private Boolean isEdge; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUPS, + type = CommandType.LIST, collectionType = CommandType.STRING, + description = "comma separated list of storage access groups for the hosts in the zone", + since = "4.21.0") + private List storageAccessGroups; ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// @@ -162,6 +169,10 @@ public class CreateZoneCmd extends BaseCmd { return isEdge; } + public List getStorageAccessGroups() { + return storageAccessGroups; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java index 0245f228b89..548f4d67b23 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/DeleteBackupScheduleCmd.java @@ -26,6 +26,7 @@ import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.BackupScheduleResponse; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.backup.BackupManager; @@ -54,10 +55,16 @@ public class DeleteBackupScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, - required = true, description = "ID of the VM") private Long vmId; + @Parameter(name = ApiConstants.ID, + type = CommandType.UUID, + entityType = BackupScheduleResponse.class, + description = "ID of the schedule", + since = "4.20.1") + private Long id; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -66,6 +73,9 @@ public class DeleteBackupScheduleCmd extends BaseCmd { return vmId; } + public Long getId() { return id; } + + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -73,7 +83,7 @@ public class DeleteBackupScheduleCmd extends BaseCmd { @Override public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { try { - boolean result = backupManager.deleteBackupSchedule(getVmId()); + boolean result = backupManager.deleteBackupSchedule(this); if (result) { SuccessResponse response = new SuccessResponse(getCommandName()); response.setResponseName(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java index 0cecbb37020..77a7a7fd8ea 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/config/ListCapabilitiesCmd.java @@ -72,6 +72,7 @@ public class ListCapabilitiesCmd extends BaseCmd { response.setInstancesDisksStatsRetentionTime((Integer) capabilities.get(ApiConstants.INSTANCES_DISKS_STATS_RETENTION_TIME)); response.setSharedFsVmMinCpuCount((Integer)capabilities.get(ApiConstants.SHAREDFSVM_MIN_CPU_COUNT)); response.setSharedFsVmMinRamSize((Integer)capabilities.get(ApiConstants.SHAREDFSVM_MIN_RAM_SIZE)); + response.setInstanceLeaseEnabled((Boolean) capabilities.get(ApiConstants.INSTANCE_LEASE_ENABLED)); response.setObjectName("capability"); response.setResponseName(getCommandName()); this.setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java index 7861c1e5d41..4d11f0dc3d2 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ExtractIsoCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.iso; +import com.cloud.dc.DataCenter; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; @@ -101,7 +102,15 @@ public class ExtractIsoCmd extends BaseAsyncCmd { @Override public String getEventDescription() { - return "extracting ISO: " + getId() + " from zone: " + getZoneId(); + String isoId = this._uuidMgr.getUuid(VirtualMachineTemplate.class, getId()); + String baseDescription = String.format("Extracting ISO: %s", isoId); + + Long zoneId = getZoneId(); + if (zoneId == null) { + return baseDescription; + } + + return String.format("%s from zone: %s", baseDescription, this._uuidMgr.getUuid(DataCenter.class, zoneId)); } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java index 0fa0679bfd9..22f59351e9a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ExtractTemplateCmd.java @@ -101,7 +101,15 @@ public class ExtractTemplateCmd extends BaseAsyncCmd { @Override public String getEventDescription() { - return "extracting template: " + this._uuidMgr.getUuid(VirtualMachineTemplate.class, getId()) + ((getZoneId() != null) ? " from zone: " + this._uuidMgr.getUuid(DataCenter.class, getZoneId()) : ""); + String templateId = this._uuidMgr.getUuid(VirtualMachineTemplate.class, getId()); + String baseDescription = String.format("Extracting template: %s", templateId); + + Long zoneId = getZoneId(); + if (zoneId == null) { + return baseDescription; + } + + return String.format("%s from zone: %s", baseDescription, this._uuidMgr.getUuid(DataCenter.class, zoneId)); } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java index 52d42a95d98..bf6e41e6d59 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java @@ -16,17 +16,24 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nonnull; - +import com.cloud.agent.api.LogLevel; +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InsufficientServerCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.network.Network; +import com.cloud.network.Network.IpAddresses; +import com.cloud.offering.DiskOffering; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.uservm.UserVm; +import com.cloud.utils.net.Dhcp; +import com.cloud.utils.net.NetUtils; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ACL; @@ -53,29 +60,22 @@ import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.vm.lease.VMLeaseManager; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; -import com.cloud.agent.api.LogLevel; -import com.cloud.event.EventTypes; -import com.cloud.exception.ConcurrentOperationException; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.InsufficientServerCapacityException; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.ResourceAllocationException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.network.Network; -import com.cloud.network.Network.IpAddresses; -import com.cloud.offering.DiskOffering; -import com.cloud.template.VirtualMachineTemplate; -import com.cloud.uservm.UserVm; -import com.cloud.utils.net.Dhcp; -import com.cloud.utils.net.NetUtils; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VmDetailConstants; +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; @APICommand(name = "deployVirtualMachine", description = "Creates and automatically starts a virtual machine based on a service offering, disk offering, and template.", responseObject = UserVmResponse.class, responseView = ResponseView.Restricted, entityType = {VirtualMachine.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = true) @@ -278,6 +278,14 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG description = "Enable packed virtqueues or not.") private Boolean nicPackedVirtQueues; + @Parameter(name = ApiConstants.INSTANCE_LEASE_DURATION, type = CommandType.INTEGER, since = "4.21.0", + description = "Number of days instance is leased for.") + private Integer leaseDuration; + + @Parameter(name = ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION, type = CommandType.STRING, since = "4.21.0", + description = "Lease expiry action, valid values are STOP and DESTROY") + private String leaseExpiryAction; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -475,6 +483,22 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG return password; } + public Integer getLeaseDuration() { + return leaseDuration; + } + + public VMLeaseManager.ExpiryAction getLeaseExpiryAction() { + if (StringUtils.isBlank(leaseExpiryAction)) { + return null; + } + VMLeaseManager.ExpiryAction action = EnumUtils.getEnumIgnoreCase(VMLeaseManager.ExpiryAction.class, leaseExpiryAction); + if (action == null) { + throw new InvalidParameterValueException("Invalid value configured for leaseexpiryaction, valid values are: " + + com.cloud.utils.EnumUtils.listValues(VMLeaseManager.ExpiryAction.values())); + } + return action; + } + public List getNetworkIds() { if (MapUtils.isNotEmpty(vAppNetworks)) { if (CollectionUtils.isNotEmpty(networkIds) || ipAddress != null || getIp6Address() != null || MapUtils.isNotEmpty(ipToNetworkList)) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java index 50e1798112d..eee16287eea 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVMsCmd.java @@ -41,13 +41,16 @@ import org.apache.cloudstack.api.response.ResourceIconResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VpcResponse; import org.apache.cloudstack.api.response.ZoneResponse; -import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; +import com.cloud.cpu.CPU; import com.cloud.exception.InvalidParameterValueException; import com.cloud.server.ResourceIcon; import com.cloud.server.ResourceTag; @@ -149,6 +152,19 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements @Parameter(name = ApiConstants.USER_DATA, type = CommandType.BOOLEAN, description = "Whether to return the VMs' user data or not. By default, user data will not be returned.", since = "4.18.0.0") private Boolean showUserData; + @Parameter(name = ApiConstants.USER_DATA_ID, type = CommandType.UUID, entityType = UserDataResponse.class, required = false, description = "the instances by userdata", since = "4.20.1") + private Long userdataId; + + @Parameter(name = ApiConstants.ARCH, type = CommandType.STRING, + description = "CPU arch of the VM", + since = "4.20.1") + private String arch; + + @Parameter(name = ApiConstants.LEASED, type = CommandType.BOOLEAN, + description = "Whether to return only leased instances", + since = "4.21.0") + private Boolean onlyLeasedInstances = false; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -243,6 +259,10 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements return CollectionUtils.isEmpty(viewDetails); } + public Long getUserdataId() { + return userdataId; + } + public EnumSet getDetails() throws InvalidParameterValueException { if (isViewDetailsEmpty()) { if (_queryService.ReturnVmStatsOnVmList.value()) { @@ -284,6 +304,14 @@ public class ListVMsCmd extends BaseListRetrieveOnlyResourceCountCmd implements return isVnf; } + public CPU.CPUArch getArch() { + return StringUtils.isBlank(arch) ? null : CPU.CPUArch.fromType(arch); + } + + public boolean getOnlyLeasedInstances() { + return BooleanUtils.toBoolean(onlyLeasedInstances); + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java index 0f5dade96d2..7906ec632a4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java @@ -16,20 +16,19 @@ // under the License. package org.apache.cloudstack.api.command.user.vm; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; import com.cloud.utils.exception.CloudRuntimeException; - -import org.apache.cloudstack.api.ApiArgValidator; -import org.apache.cloudstack.api.response.UserDataResponse; - +import com.cloud.utils.net.Dhcp; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -40,15 +39,17 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.user.UserCmd; import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; +import org.apache.cloudstack.api.response.UserDataResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.vm.lease.VMLeaseManager; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; -import com.cloud.exception.InsufficientCapacityException; -import com.cloud.exception.ResourceUnavailableException; -import com.cloud.user.Account; -import com.cloud.uservm.UserVm; -import com.cloud.utils.net.Dhcp; -import com.cloud.vm.VirtualMachine; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @APICommand(name = "updateVirtualMachine", description="Updates properties of a virtual machine. The VM has to be stopped and restarted for the " + "new properties to take effect. UpdateVirtualMachine does not first check whether the VM is stopped. " + @@ -154,6 +155,14 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction, " autoscaling groups or CKS, delete protection will be ignored.") private Boolean deleteProtection; + @Parameter(name = ApiConstants.INSTANCE_LEASE_DURATION, type = CommandType.INTEGER, since = "4.21.0", + description = "Number of days to lease the instance from now onward. Use -1 to remove the existing lease") + private Integer leaseDuration; + + @Parameter(name = ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION, type = CommandType.STRING, since = "4.21.0", + description = "Lease expiry action, valid values are STOP and DESTROY") + private String leaseExpiryAction; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -324,4 +333,21 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction, public ApiCommandResourceType getApiResourceType() { return ApiCommandResourceType.VirtualMachine; } + + public Integer getLeaseDuration() { + return leaseDuration; + } + + public VMLeaseManager.ExpiryAction getLeaseExpiryAction() { + if (StringUtils.isBlank(leaseExpiryAction)) { + return null; + } + VMLeaseManager.ExpiryAction action = EnumUtils.getEnumIgnoreCase(VMLeaseManager.ExpiryAction.class, leaseExpiryAction); + if (action == null) { + throw new InvalidParameterValueException("Invalid value configured for leaseexpiryaction, valid values are: " + + com.cloud.utils.EnumUtils.listValues(VMLeaseManager.ExpiryAction.values())); + } + return action; + } + } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java index 9445aba23c0..6ddb5f1e567 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/ExtractVolumeCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.volume; +import com.cloud.dc.DataCenter; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; @@ -114,12 +115,15 @@ public class ExtractVolumeCmd extends BaseAsyncCmd { @Override public String getEventDescription() { - return "Extraction job"; + String volumeId = this._uuidMgr.getUuid(Volume.class, getId()); + String zoneId = this._uuidMgr.getUuid(DataCenter.class, getZoneId()); + + return String.format("Extracting volume: %s from zone: %s", volumeId, zoneId); } @Override public void execute() { - CallContext.current().setEventDetails("Volume Id: " + this._uuidMgr.getUuid(Volume.class, getId())); + CallContext.current().setEventDetails(getEventDescription()); String uploadUrl = _volumeService.extractVolume(this); if (uploadUrl != null) { ExtractResponse response = _responseGenerator.createVolumeExtractResponse(id, zoneId, getEntityOwnerId(), mode, uploadUrl); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java index d926257437e..a5e26f30dfb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/zone/ListZonesCmd.java @@ -69,6 +69,11 @@ public class ListZonesCmd extends BaseListCmd implements UserCmd { @Parameter(name = ApiConstants.SHOW_RESOURCE_ICON, type = CommandType.BOOLEAN, description = "flag to display the resource image for the zones") private Boolean showIcon; + @Parameter(name = ApiConstants.STORAGE_ACCESS_GROUP, type = CommandType.STRING, + description = "the name of the storage access group", + since = "4.21.0") + private String storageAccessGroup; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -109,6 +114,18 @@ public class ListZonesCmd extends BaseListCmd implements UserCmd { return showIcon != null ? showIcon : false; } + public String getStorageAccessGroup() { + return storageAccessGroup; + } + + public ListZonesCmd() { + + } + + public ListZonesCmd(String storageAccessGroup) { + this.storageAccessGroup = storageAccessGroup; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java index 3847176608c..0db51f04034 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/BackupRepositoryResponse.java @@ -57,10 +57,6 @@ public class BackupRepositoryResponse extends BaseResponse { @Param(description = "backup type") private String type; - @SerializedName(ApiConstants.MOUNT_OPTIONS) - @Param(description = "mount options for the backup repository") - private String mountOptions; - @SerializedName(ApiConstants.CAPACITY_BYTES) @Param(description = "capacity of the backup repository") private Long capacityBytes; @@ -112,14 +108,6 @@ public class BackupRepositoryResponse extends BaseResponse { this.address = address; } - public String getMountOptions() { - return mountOptions; - } - - public void setMountOptions(String mountOptions) { - this.mountOptions = mountOptions; - } - public String getProviderName() { return providerName; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java index 3861ac455ed..74dbfa15a43 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/CapabilitiesResponse.java @@ -136,6 +136,10 @@ public class CapabilitiesResponse extends BaseResponse { @Param(description = "the min Ram size for the service offering used by the shared filesystem instance", since = "4.20.0") private Integer sharedFsVmMinRamSize; + @SerializedName(ApiConstants.INSTANCE_LEASE_ENABLED) + @Param(description = "true if instance lease feature is enabled", since = "4.21.0") + private Boolean instanceLeaseEnabled; + public void setSecurityGroupsEnabled(boolean securityGroupsEnabled) { this.securityGroupsEnabled = securityGroupsEnabled; } @@ -247,4 +251,8 @@ public class CapabilitiesResponse extends BaseResponse { public void setSharedFsVmMinRamSize(Integer sharedFsVmMinRamSize) { this.sharedFsVmMinRamSize = sharedFsVmMinRamSize; } + + public void setInstanceLeaseEnabled(Boolean instanceLeaseEnabled) { + this.instanceLeaseEnabled = instanceLeaseEnabled; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ClusterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ClusterResponse.java index 1c69849239f..17c86072b98 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ClusterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ClusterResponse.java @@ -95,6 +95,18 @@ public class ClusterResponse extends BaseResponseWithAnnotations { @Param(description = "CPU Arch of the hosts in the cluster", since = "4.20") private String arch; + @SerializedName(ApiConstants.STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups for the host", since = "4.21.0") + private String storageAccessGroups; + + @SerializedName(ApiConstants.POD_STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups on the pod", since = "4.21.0") + private String podStorageAccessGroups; + + @SerializedName(ApiConstants.ZONE_STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups on the zone", since = "4.21.0") + private String zoneStorageAccessGroups; + public String getId() { return id; } @@ -259,4 +271,28 @@ public class ClusterResponse extends BaseResponseWithAnnotations { public String getArch() { return arch; } + + public String getStorageAccessGroups() { + return storageAccessGroups; + } + + public void setStorageAccessGroups(String storageAccessGroups) { + this.storageAccessGroups = storageAccessGroups; + } + + public String getPodStorageAccessGroups() { + return podStorageAccessGroups; + } + + public void setPodStorageAccessGroups(String podStorageAccessGroups) { + this.podStorageAccessGroups = podStorageAccessGroups; + } + + public String getZoneStorageAccessGroups() { + return zoneStorageAccessGroups; + } + + public void setZoneStorageAccessGroups(String zoneStorageAccessGroups) { + this.zoneStorageAccessGroups = zoneStorageAccessGroups; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java index b23d0f4b527..4b5e886a16e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/DomainRouterResponse.java @@ -245,6 +245,10 @@ public class DomainRouterResponse extends BaseResponseWithAnnotations implements @Param(description = "the version of the code / software in the router") private String softwareVersion; + @SerializedName(ApiConstants.ARCH) + @Param(description = "CPU arch of the router", since = "4.20.1") + private String arch; + public DomainRouterResponse() { nics = new LinkedHashSet(); } @@ -518,4 +522,8 @@ public class DomainRouterResponse extends BaseResponseWithAnnotations implements public void setSoftwareVersion(String softwareVersion) { this.softwareVersion = softwareVersion; } + + public void setArch(String arch) { + this.arch = arch; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java index c9a5c47887d..342a1eb7df3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java @@ -152,7 +152,7 @@ public class HostResponse extends BaseResponseWithAnnotations { @Deprecated @SerializedName("memoryallocated") @Param(description = "the amount of the host's memory currently allocated") - private Long memoryAllocated; + private long memoryAllocated; @SerializedName("memoryallocatedpercentage") @Param(description = "the amount of the host's memory currently allocated in percentage") @@ -302,6 +302,22 @@ public class HostResponse extends BaseResponseWithAnnotations { @Param(description = "CPU Arch of the host", since = "4.20") private String arch; + @SerializedName(ApiConstants.STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups for the host", since = "4.21.0") + private String storageAccessGroups; + + @SerializedName(ApiConstants.CLUSTER_STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups on the cluster", since = "4.21.0") + private String clusterStorageAccessGroups; + + @SerializedName(ApiConstants.POD_STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups on the pod", since = "4.21.0") + private String podStorageAccessGroups; + + @SerializedName(ApiConstants.ZONE_STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups on the zone", since = "4.21.0") + private String zoneStorageAccessGroups; + @Override public String getObjectId() { return this.getId(); @@ -415,7 +431,7 @@ public class HostResponse extends BaseResponseWithAnnotations { this.memWithOverprovisioning=memWithOverprovisioning; } - public void setMemoryAllocated(Long memoryAllocated) { + public void setMemoryAllocated(long memoryAllocated) { this.memoryAllocated = memoryAllocated; } @@ -491,6 +507,38 @@ public class HostResponse extends BaseResponseWithAnnotations { this.hostTags = hostTags; } + public String getStorageAccessGroups() { + return storageAccessGroups; + } + + public void setStorageAccessGroups(String storageAccessGroups) { + this.storageAccessGroups = storageAccessGroups; + } + + public String getClusterStorageAccessGroups() { + return clusterStorageAccessGroups; + } + + public void setClusterStorageAccessGroups(String clusterStorageAccessGroups) { + this.clusterStorageAccessGroups = clusterStorageAccessGroups; + } + + public String getPodStorageAccessGroups() { + return podStorageAccessGroups; + } + + public void setPodStorageAccessGroups(String podStorageAccessGroups) { + this.podStorageAccessGroups = podStorageAccessGroups; + } + + public String getZoneStorageAccessGroups() { + return zoneStorageAccessGroups; + } + + public void setZoneStorageAccessGroups(String zoneStorageAccessGroups) { + this.zoneStorageAccessGroups = zoneStorageAccessGroups; + } + public String getExplicitHostTags() { return explicitHostTags; } @@ -703,8 +751,8 @@ public class HostResponse extends BaseResponseWithAnnotations { return memoryTotal; } - public Long getMemoryAllocated() { - return memoryAllocated == null ? 0 : memoryAllocated; + public long getMemoryAllocated() { + return memoryAllocated; } public void setMemoryAllocatedPercentage(String memoryAllocatedPercentage) { diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java index 729fb5ff3bc..8592a762c61 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ManagementServerResponse.java @@ -74,6 +74,11 @@ public class ManagementServerResponse extends BaseResponse { @Param(description = "the running OS kernel version for this Management Server") private String kernelVersion; + @Deprecated + @SerializedName(ApiConstants.SERVICE_IP) + @Param(description = "the IP Address for this Management Server. This is deprecated, please use 'ipaddress' instead.") + private String serviceIp; + @SerializedName(ApiConstants.IP_ADDRESS) @Param(description = "the IP Address for this Management Server") private String ipAddress; @@ -82,6 +87,14 @@ public class ManagementServerResponse extends BaseResponse { @Param(description = "the Management Server Peers") private List peers; + @SerializedName(ApiConstants.LAST_AGENTS) + @Param(description = "the last agents this Management Server is responsible for, before shutdown or preparing for maintenance", since = "4.21.0.0") + private List lastAgents; + + @SerializedName(ApiConstants.AGENTS) + @Param(description = "the agents this Management Server is responsible for", since = "4.21.0.0") + private List agents; + @SerializedName(ApiConstants.AGENTS_COUNT) @Param(description = "the number of host agents this Management Server is responsible for", since = "4.21.0.0") private Long agentsCount; @@ -130,10 +143,22 @@ public class ManagementServerResponse extends BaseResponse { return lastBoot; } + public String getServiceIp() { + return serviceIp; + } + public String getIpAddress() { return ipAddress; } + public List getLastAgents() { + return lastAgents; + } + + public List getAgents() { + return agents; + } + public Long getAgentsCount() { return this.agentsCount; } @@ -186,10 +211,22 @@ public class ManagementServerResponse extends BaseResponse { this.kernelVersion = kernelVersion; } + public void setServiceIp(String serviceIp) { + this.serviceIp = serviceIp; + } + public void setIpAddress(String ipAddress) { this.ipAddress = ipAddress; } + public void setLastAgents(List lastAgents) { + this.lastAgents = lastAgents; + } + + public void setAgents(List agents) { + this.agents = agents; + } + public void setAgentsCount(Long agentsCount) { this.agentsCount = agentsCount; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java index db811ffbe2d..a1ffda72234 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java @@ -28,12 +28,11 @@ import org.apache.cloudstack.api.BaseResponseWithAssociatedNetwork; import org.apache.cloudstack.api.EntityReference; import com.cloud.network.Network; -import com.cloud.projects.ProjectAccount; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; @SuppressWarnings("unused") -@EntityReference(value = {Network.class, ProjectAccount.class}) +@EntityReference(value = {Network.class}) public class NetworkResponse extends BaseResponseWithAssociatedNetwork implements ControlledEntityResponse, SetResourceIconResponse { @SerializedName(ApiConstants.ID) diff --git a/api/src/main/java/org/apache/cloudstack/api/response/PodResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/PodResponse.java index 587fabfae8d..6a1afaecbcf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/PodResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/PodResponse.java @@ -85,6 +85,14 @@ public class PodResponse extends BaseResponseWithAnnotations { @Param(description = "the capacity of the Pod", responseObject = CapacityResponse.class) private List capacities; + @SerializedName(ApiConstants.STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups for the pod", since = "4.21.0") + private String storageAccessGroups; + + @SerializedName(ApiConstants.ZONE_STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups on the zone", since = "4.21.0") + private String zoneStorageAccessGroups; + public String getId() { return id; } @@ -184,4 +192,20 @@ public class PodResponse extends BaseResponseWithAnnotations { public void setCapacities(List capacities) { this.capacities = capacities; } + + public String getStorageAccessGroups() { + return storageAccessGroups; + } + + public void setStorageAccessGroups(String storageAccessGroups) { + this.storageAccessGroups = storageAccessGroups; + } + + public String getZoneStorageAccessGroups() { + return zoneStorageAccessGroups; + } + + public void setZoneStorageAccessGroups(String zoneStorageAccessGroups) { + this.zoneStorageAccessGroups = zoneStorageAccessGroups; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java index 0622b936f6e..ca9358e2e5e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ServiceOfferingResponse.java @@ -80,7 +80,7 @@ public class ServiceOfferingResponse extends BaseResponseWithAnnotations { @Param(description = "true if the vm needs to be volatile, i.e., on every reboot of vm from API root disk is discarded and creates a new root disk") private Boolean isVolatile; - @SerializedName("storagetags") + @SerializedName(ApiConstants.STORAGE_TAGS) @Param(description = "the tags for the service offering") private String tags; @@ -238,6 +238,14 @@ public class ServiceOfferingResponse extends BaseResponseWithAnnotations { @Param(description = "Whether to cleanup VM and its associated resource upon expunge", since = "4.20") private Boolean purgeResources; + @SerializedName(ApiConstants.INSTANCE_LEASE_DURATION) + @Param(description = "Instance lease duration (in days) for service offering", since = "4.21.0") + private Integer leaseDuration; + + @SerializedName(ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION) + @Param(description = "Action to be taken once lease is over", since = "4.21.0") + private String leaseExpiryAction; + public ServiceOfferingResponse() { } @@ -505,6 +513,22 @@ public class ServiceOfferingResponse extends BaseResponseWithAnnotations { this.cacheMode = cacheMode; } + public Integer getLeaseDuration() { + return leaseDuration; + } + + public void setLeaseDuration(Integer leaseDuration) { + this.leaseDuration = leaseDuration; + } + + public String getLeaseExpiryAction() { + return leaseExpiryAction; + } + + public void setLeaseExpiryAction(String leaseExpiryAction) { + this.leaseExpiryAction = leaseExpiryAction; + } + public String getVsphereStoragePolicy() { return vsphereStoragePolicy; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StorageAccessGroupResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StorageAccessGroupResponse.java new file mode 100644 index 00000000000..a6324dd62a9 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/response/StorageAccessGroupResponse.java @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; +import com.cloud.serializer.Param; + +public class StorageAccessGroupResponse extends BaseResponse { + @SerializedName(ApiConstants.ID) + @Param(description = "the ID of the storage access group") + private String id; + + @SerializedName(ApiConstants.NAME) + @Param(description = "the name of the storage access group") + private String name; + + @SerializedName("hosts") + @Param(description = "List of Hosts in the Storage Access Group") + private ListResponse hostResponseList; + + @SerializedName("clusters") + @Param(description = "List of Clusters in the Storage Access Group") + private ListResponse clusterResponseList; + + @SerializedName("pods") + @Param(description = "List of Pods in the Storage Access Group") + private ListResponse podResponseList; + + @SerializedName("zones") + @Param(description = "List of Zones in the Storage Access Group") + private ListResponse zoneResponseList; + + @SerializedName("storagepools") + @Param(description = "List of Storage Pools in the Storage Access Group") + private ListResponse storagePoolResponseList; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public ListResponse getHostResponseList() { + return hostResponseList; + } + + public void setHostResponseList(ListResponse hostResponseList) { + this.hostResponseList = hostResponseList; + } + + public ListResponse getClusterResponseList() { + return clusterResponseList; + } + + public void setClusterResponseList(ListResponse clusterResponseList) { + this.clusterResponseList = clusterResponseList; + } + + public ListResponse getPodResponseList() { + return podResponseList; + } + + public void setPodResponseList(ListResponse podResponseList) { + this.podResponseList = podResponseList; + } + + public ListResponse getZoneResponseList() { + return zoneResponseList; + } + + public void setZoneResponseList(ListResponse zoneResponseList) { + this.zoneResponseList = zoneResponseList; + } + + public ListResponse getStoragePoolResponseList() { + return storagePoolResponseList; + } + + public void setStoragePoolResponseList(ListResponse storagePoolResponseList) { + this.storagePoolResponseList = storagePoolResponseList; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java index 676803ea86b..abc674ff0f9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/StoragePoolResponse.java @@ -109,6 +109,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "the tags for the storage pool") private String tags; + @SerializedName(ApiConstants.STORAGE_ACCESS_GROUPS) + @Param(description = "the storage access groups for the storage pool", since = "4.21.0") + private String storageAccessGroups; + @SerializedName(ApiConstants.NFS_MOUNT_OPTIONS) @Param(description = "the nfs mount options for the storage pool", since = "4.19.1") private String nfsMountOpts; @@ -149,6 +153,10 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { @Param(description = "whether this pool is managed or not") private Boolean managed; + @SerializedName(ApiConstants.DETAILS) + @Param(description = "the storage pool details") + private Map details; + public Map getCaps() { return caps; } @@ -340,6 +348,14 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { this.tags = tags; } + public String getStorageAccessGroups() { + return storageAccessGroups; + } + + public void setStorageAccessGroups(String storageAccessGroups) { + this.storageAccessGroups = storageAccessGroups; + } + public Boolean getIsTagARule() { return isTagARule; } @@ -407,4 +423,12 @@ public class StoragePoolResponse extends BaseResponseWithAnnotations { public void setManaged(Boolean managed) { this.managed = managed; } + + public Map getDetails() { + return details; + } + + public void setDetails(Map details) { + this.details = details; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java index 31a8b731491..be9f14f5060 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/SystemVmResponse.java @@ -186,6 +186,10 @@ public class SystemVmResponse extends BaseResponseWithAnnotations { @Param(description = "the name of the service offering of the system virtual machine.") private String serviceOfferingName; + @SerializedName(ApiConstants.ARCH) + @Param(description = "CPU arch of the system VM", since = "4.20.1") + private String arch; + @Override public String getObjectId() { return this.getId(); @@ -490,4 +494,8 @@ public class SystemVmResponse extends BaseResponseWithAnnotations { public void setServiceOfferingName(String serviceOfferingName) { this.serviceOfferingName = serviceOfferingName; } + + public void setArch(String arch) { + this.arch = arch; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java index 7a26b178591..c1156f5f23a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java @@ -79,6 +79,14 @@ public class UnmanagedInstanceResponse extends BaseResponse { @Param(description = "the operating system of the virtual machine") private String operatingSystem; + @SerializedName(ApiConstants.BOOT_MODE) + @Param(description = "indicates the boot mode") + private String bootMode; + + @SerializedName(ApiConstants.BOOT_TYPE) + @Param(description = "indicates the boot type") + private String bootType; + @SerializedName(ApiConstants.DISK) @Param(description = "the list of disks associated with the virtual machine", responseObject = UnmanagedInstanceDiskResponse.class) private Set disks; @@ -211,4 +219,20 @@ public class UnmanagedInstanceResponse extends BaseResponse { public void addNic(NicResponse nic) { this.nics.add(nic); } + + public String getBootMode() { + return bootMode; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + + public String getBootType() { + return bootType; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java index 1f4b493fba2..d873bc65709 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java @@ -31,13 +31,13 @@ import org.apache.cloudstack.affinity.AffinityGroupResponse; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponseWithTagInformation; import org.apache.cloudstack.api.EntityReference; +import org.apache.commons.collections.CollectionUtils; import com.cloud.network.router.VirtualRouter; import com.cloud.serializer.Param; import com.cloud.uservm.UserVm; import com.cloud.vm.VirtualMachine; import com.google.gson.annotations.SerializedName; -import org.apache.commons.collections.CollectionUtils; @SuppressWarnings("unused") @EntityReference(value = {VirtualMachine.class, UserVm.class, VirtualRouter.class}) @@ -392,10 +392,26 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co @Param(description = "VNF details", since = "4.19.0") private Map vnfDetails; - @SerializedName((ApiConstants.VM_TYPE)) + @SerializedName(ApiConstants.VM_TYPE) @Param(description = "User VM type", since = "4.20.0") private String vmType; + @SerializedName(ApiConstants.ARCH) + @Param(description = "CPU arch of the VM", since = "4.20.1") + private String arch; + + @SerializedName(ApiConstants.INSTANCE_LEASE_DURATION) + @Param(description = "Instance lease duration in days", since = "4.21.0") + private Integer leaseDuration; + + @SerializedName(ApiConstants.INSTANCE_LEASE_EXPIRY_DATE) + @Param(description = "Instance lease expiry date", since = "4.21.0") + private Date leaseExpiryDate; + + @SerializedName(ApiConstants.INSTANCE_LEASE_EXPIRY_ACTION) + @Param(description = "Instance lease expiry action", since = "4.21.0") + private String leaseExpiryAction; + public UserVmResponse() { securityGroupList = new LinkedHashSet<>(); nics = new TreeSet<>(Comparator.comparingInt(x -> Integer.parseInt(x.getDeviceId()))); @@ -1169,4 +1185,37 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co public void setIpAddress(String ipAddress) { this.ipAddress = ipAddress; } + + public String getArch() { + return arch; + } + + public void setArch(String arch) { + this.arch = arch; + } + + public Integer getLeaseDuration() { + return leaseDuration; + } + + public void setLeaseDuration(Integer leaseDuration) { + this.leaseDuration = leaseDuration; + } + + public String getLeaseExpiryAction() { + return leaseExpiryAction; + } + + public void setLeaseExpiryAction(String leaseExpiryAction) { + this.leaseExpiryAction = leaseExpiryAction; + } + + public Date getLeaseExpiryDate() { + return leaseExpiryDate; + } + + public void setLeaseExpiryDate(Date leaseExpiryDate) { + this.leaseExpiryDate = leaseExpiryDate; + } + } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java index 4a5279753a1..8e9a993bac6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ZoneResponse.java @@ -95,7 +95,7 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @SerializedName("securitygroupsenabled") @Param(description = "true if security groups support is enabled, false otherwise") - private boolean securityGroupsEnabled; + private Boolean securityGroupsEnabled; @SerializedName("allocationstate") @Param(description = "the allocation state of the cluster") @@ -115,7 +115,7 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @SerializedName(ApiConstants.LOCAL_STORAGE_ENABLED) @Param(description = "true if local storage offering enabled, false otherwise") - private boolean localStorageEnabled; + private Boolean localStorageEnabled; @SerializedName(ApiConstants.TAGS) @Param(description = "the list of resource tags associated with zone.", responseObject = ResourceTagResponse.class, since = "4.3") @@ -161,11 +161,19 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso @Param(description = "true, if routed network/vpc is enabled", since = "4.20.1") private boolean routedModeEnabled = false; + @SerializedName(ApiConstants.STORAGE_ACCESS_GROUPS) + @Param(description = "comma-separated list of storage access groups for the zone", since = "4.21.0") + private String storageAccessGroups; + public ZoneResponse() { tags = new LinkedHashSet(); } + public ZoneResponse(Set tags) { + this.tags = tags; + } + public void setId(String id) { this.id = id; } @@ -402,6 +410,14 @@ public class ZoneResponse extends BaseResponseWithAnnotations implements SetReso return type; } + public String getStorageAccessGroups() { + return storageAccessGroups; + } + + public void setStorageAccessGroups(String storageAccessGroups) { + this.storageAccessGroups = storageAccessGroups; + } + public void setNsxEnabled(boolean nsxEnabled) { this.nsxEnabled = nsxEnabled; } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index cbd4b7e0596..eebad3af067 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -23,6 +23,7 @@ import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.command.admin.backup.ImportBackupOfferingCmd; import org.apache.cloudstack.api.command.admin.backup.UpdateBackupOfferingCmd; import org.apache.cloudstack.api.command.user.backup.CreateBackupScheduleCmd; +import org.apache.cloudstack.api.command.user.backup.DeleteBackupScheduleCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupOfferingsCmd; import org.apache.cloudstack.api.command.user.backup.ListBackupsCmd; import org.apache.cloudstack.framework.config.ConfigKey; @@ -192,10 +193,10 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer /** * Deletes VM backup schedule for a VM - * @param vmId + * @param cmd * @return */ - boolean deleteBackupSchedule(Long vmId); + boolean deleteBackupSchedule(DeleteBackupScheduleCmd cmd); /** * Creates backup of a VM diff --git a/api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java b/api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java new file mode 100644 index 00000000000..89ab97990df --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/command/ReconcileCommandService.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.hypervisor.Hypervisor; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.framework.config.ConfigKey; + +import java.util.Arrays; +import java.util.List; + +public interface ReconcileCommandService { + + ConfigKey ReconcileCommandsEnabled = new ConfigKey<>("Advanced", Boolean.class, + "reconcile.commands.enabled", "false", + "Indicates whether the background task to reconcile the commands is enabled or not", + false); + + ConfigKey ReconcileCommandsInterval = new ConfigKey<>("Advanced", Integer.class, + "reconcile.commands.interval", "60", + "Interval (in seconds) for the background task to reconcile the commands", + false); + ConfigKey ReconcileCommandsMaxAttempts = new ConfigKey<>("Advanced", Integer.class, + "reconcile.commands.max.attempts", "30", + "The maximum number of attempts to reconcile the commands", + true); + + ConfigKey ReconcileCommandsWorkers = new ConfigKey<>("Advanced", Integer.class, + "reconcile.commands.workers", "100", + "The Number of worker threads to reconcile the commands", + false); + + List SupportedHypervisorTypes = Arrays.asList(Hypervisor.HypervisorType.KVM); + + void persistReconcileCommands(Long hostId, Long requestSequence, Command[] cmd); + + boolean updateReconcileCommand(long requestSeq, Command command, Answer answer, Command.State newStateByManagement, Command.State newStateByAgent); + + void processCommand(Command pingCommand, Answer pingAnswer); + + void processAnswers(long requestSeq, Command[] commands, Answer[] answers); + + void updateReconcileCommandToInterruptedByManagementServerId(long managementServerId); + + void updateReconcileCommandToInterruptedByHostId(long hostId); + + boolean isReconcileResourceNeeded(long resourceId, ApiCommandResourceType resourceType); +} diff --git a/api/src/main/java/org/apache/cloudstack/query/QueryService.java b/api/src/main/java/org/apache/cloudstack/query/QueryService.java index 88081494320..4278c9217b5 100644 --- a/api/src/main/java/org/apache/cloudstack/query/QueryService.java +++ b/api/src/main/java/org/apache/cloudstack/query/QueryService.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.query; +import java.util.Arrays; import java.util.List; import org.apache.cloudstack.affinity.AffinityGroupResponse; @@ -31,6 +32,7 @@ import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListObjectStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; +import org.apache.cloudstack.api.command.admin.storage.ListStorageAccessGroupsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; import org.apache.cloudstack.api.command.admin.storage.heuristics.ListSecondaryStorageSelectorsCmd; @@ -86,6 +88,7 @@ import org.apache.cloudstack.api.response.SecondaryStorageHeuristicsResponse; import org.apache.cloudstack.api.response.SecurityGroupResponse; import org.apache.cloudstack.api.response.ServiceOfferingResponse; import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StorageAccessGroupResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.StorageTagResponse; import org.apache.cloudstack.api.response.TemplateResponse; @@ -97,6 +100,7 @@ import org.apache.cloudstack.api.response.ZoneResponse; import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.exception.PermissionDeniedException; +import com.cloud.vm.VmDetailConstants; /** * Service used for list api query. @@ -104,6 +108,8 @@ import com.cloud.exception.PermissionDeniedException; */ public interface QueryService { + List RootAdminOnlyVmSettings = Arrays.asList(VmDetailConstants.GUEST_CPU_MODE, VmDetailConstants.GUEST_CPU_MODEL); + // Config keys ConfigKey AllowUserViewDestroyedVM = new ConfigKey<>("Advanced", Boolean.class, "allow.user.view.destroyed.vm", "false", "Determines whether users can view their destroyed or expunging vm ", true, ConfigKey.Scope.Account); @@ -193,6 +199,8 @@ public interface QueryService { ListResponse searchForStorageTags(ListStorageTagsCmd cmd); + ListResponse searchForStorageAccessGroups(ListStorageAccessGroupsCmd cmd); + ListResponse searchForHostTags(ListHostTagsCmd cmd); ListResponse listManagementServers(ListMgmtsCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index 3d5646f68c9..bba97dff71c 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -61,6 +61,9 @@ public class UnmanagedInstanceTO { private String vncPassword; + private String bootType; + private String bootMode; + public String getName() { return name; } @@ -196,6 +199,22 @@ public class UnmanagedInstanceTO { this, "name", "internalCSName", "hostName", "clusterName")); } + public String getBootType() { + return bootType; + } + + public void setBootType(String bootType) { + this.bootType = bootType; + } + + public String getBootMode() { + return bootMode; + } + + public void setBootMode(String bootMode) { + this.bootMode = bootMode; + } + public static class Disk { private String diskId; diff --git a/api/src/main/java/org/apache/cloudstack/vm/lease/VMLeaseManager.java b/api/src/main/java/org/apache/cloudstack/vm/lease/VMLeaseManager.java new file mode 100644 index 00000000000..b6702695305 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/vm/lease/VMLeaseManager.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cloudstack.vm.lease; + +import com.cloud.utils.component.Manager; +import org.apache.cloudstack.framework.config.ConfigKey; + +import java.util.List; + +public interface VMLeaseManager extends Manager { + + int MAX_LEASE_DURATION_DAYS = 365_00; // 100 years + + enum ExpiryAction { + STOP, + DESTROY + } + + enum LeaseActionExecution { + PENDING, + DISABLED, + DONE, + CANCELLED + } + + ConfigKey InstanceLeaseEnabled = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Boolean.class, + "instance.lease.enabled", "false", "Indicates whether to enable the Instance lease," + + " will be applicable only on instances created after lease is enabled. Disabling the feature cancels lease on existing instances with lease." + + " Re-enabling feature will not cause lease expiry actions on grandfathered instances", + true, List.of(ConfigKey.Scope.Global)); + + ConfigKey InstanceLeaseSchedulerInterval = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, + "instance.lease.scheduler.interval", "3600", "VM Lease Scheduler interval in seconds", + false, List.of(ConfigKey.Scope.Global)); + + ConfigKey InstanceLeaseExpiryEventSchedulerInterval = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, + "instance.lease.eventscheduler.interval", "86400", "Lease expiry event Scheduler interval in seconds", + false, List.of(ConfigKey.Scope.Global)); + + ConfigKey InstanceLeaseExpiryEventDaysBefore = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, + "instance.lease.expiryevent.daysbefore", "7", "Indicates how many days in advance, expiry events will be created before expiry.", + true, List.of(ConfigKey.Scope.Global)); + + void onLeaseFeatureToggle(); +} diff --git a/api/src/test/java/com/cloud/cpu/CPUTest.java b/api/src/test/java/com/cloud/cpu/CPUTest.java new file mode 100644 index 00000000000..dfedf21864c --- /dev/null +++ b/api/src/test/java/com/cloud/cpu/CPUTest.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.cpu; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class CPUTest { + @Test + public void testCPUArchGetType() { + assertEquals("i686", CPU.CPUArch.x86.getType()); + assertEquals("x86_64", CPU.CPUArch.amd64.getType()); + assertEquals("aarch64", CPU.CPUArch.arm64.getType()); + } + + @Test + public void testCPUArchGetBits() { + assertEquals(32, CPU.CPUArch.x86.getBits()); + assertEquals(64, CPU.CPUArch.amd64.getBits()); + assertEquals(64, CPU.CPUArch.arm64.getBits()); + } + + @Test + public void testCPUArchFromTypeWithValidValues() { + assertEquals(CPU.CPUArch.x86, CPU.CPUArch.fromType("i686")); + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType("x86_64")); + assertEquals(CPU.CPUArch.arm64, CPU.CPUArch.fromType("aarch64")); + } + + @Test + public void testCPUArchFromTypeWithDefaultForBlankOrNull() { + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType("")); + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType(" ")); + assertEquals(CPU.CPUArch.amd64, CPU.CPUArch.fromType(null)); + } + + @Test + public void testCPUArchFromTypeWithInvalidValue() { + Exception exception = assertThrows(IllegalArgumentException.class, () -> { + CPU.CPUArch.fromType("unsupported"); + }); + assertTrue(exception.getMessage().contains("Unsupported arch type: unsupported")); + } + + @Test + public void testCPUArchGetTypesAsCSV() { + String expectedCSV = "i686,x86_64,aarch64"; + assertEquals(expectedCSV, CPU.CPUArch.getTypesAsCSV()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java index 6daa5de07cb..d6f9d9b6937 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/offering/CreateServiceOfferingCmdTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.command.admin.offering; +import com.cloud.exception.InvalidParameterValueException; +import org.apache.cloudstack.vm.lease.VMLeaseManager; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -55,4 +57,25 @@ public class CreateServiceOfferingCmdTest { ReflectionTestUtils.setField(createServiceOfferingCmd, "purgeResources", true); Assert.assertTrue(createServiceOfferingCmd.isPurgeResources()); } + + @Test + public void testGetLeaseDuration() { + ReflectionTestUtils.setField(createServiceOfferingCmd, "leaseDuration", 10); + Assert.assertEquals(10, createServiceOfferingCmd.getLeaseDuration().longValue()); + } + + @Test + public void testGetLeaseExpiryAction() { + ReflectionTestUtils.setField(createServiceOfferingCmd, "leaseExpiryAction", "stop"); + Assert.assertEquals(VMLeaseManager.ExpiryAction.STOP, createServiceOfferingCmd.getLeaseExpiryAction()); + + ReflectionTestUtils.setField(createServiceOfferingCmd, "leaseExpiryAction", "DESTROY"); + Assert.assertEquals(VMLeaseManager.ExpiryAction.DESTROY, createServiceOfferingCmd.getLeaseExpiryAction()); + } + + @Test(expected = InvalidParameterValueException.class) + public void testGetLeaseExpiryActionInvalidValue() { + ReflectionTestUtils.setField(createServiceOfferingCmd, "leaseExpiryAction", "Unknown"); + Assert.assertEquals(null, createServiceOfferingCmd.getLeaseExpiryAction()); + } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java index a628f13275c..c8ca1c2afe3 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/resource/PurgeExpungedResourcesCmdTest.java @@ -44,7 +44,7 @@ public class PurgeExpungedResourcesCmdTest { @Spy @InjectMocks - PurgeExpungedResourcesCmd spyCmd = Mockito.spy(new PurgeExpungedResourcesCmd()); + PurgeExpungedResourcesCmd spyCmd; @Test public void testGetResourceType() { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java index 98435bf6e38..ad95ce10bd6 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java @@ -20,14 +20,11 @@ package org.apache.cloudstack.api.command.admin.storage; import com.cloud.utils.Pair; import org.apache.cloudstack.api.response.ExtractResponse; import org.apache.cloudstack.storage.browser.StorageBrowser; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; @@ -48,18 +45,6 @@ public class DownloadImageStoreObjectCmdTest { @Spy private DownloadImageStoreObjectCmd cmd; - private AutoCloseable closeable; - - @Before - public void setUp() { - closeable = MockitoAnnotations.openMocks(this); - } - - @After - public void tearDown() throws Exception { - closeable.close(); - } - @Test public void testExecute() throws Exception { ReflectionTestUtils.setField(cmd, "storeId", 1L); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmdTest.java index 61a3c8fb9e6..4fec90217fd 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vm/MigrateVirtualMachineWithVolumeCmdTest.java @@ -68,6 +68,12 @@ public class MigrateVirtualMachineWithVolumeCmdTest { @Mock Host hostMock; + @Mock + private Object job; + + @Mock + private Object _responseObject; + @Spy @InjectMocks MigrateVirtualMachineWithVolumeCmd cmdSpy; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCCmdByAdminTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCCmdByAdminTest.java index c4e21bb948b..aaa65e10ff4 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCCmdByAdminTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCCmdByAdminTest.java @@ -20,7 +20,6 @@ package org.apache.cloudstack.api.command.admin.vpc; import com.cloud.network.vpc.VpcService; import com.cloud.user.AccountService; import com.cloud.utils.db.EntityManager; -import junit.framework.TestCase; import org.apache.cloudstack.api.ResponseGenerator; import org.junit.Assert; import org.junit.Test; @@ -34,7 +33,7 @@ import org.springframework.test.util.ReflectionTestUtils; import java.util.List; @RunWith(MockitoJUnitRunner.class) -public class CreateVPCCmdByAdminTest extends TestCase { +public class CreateVPCCmdByAdminTest { @Mock public VpcService _vpcService; @@ -43,8 +42,10 @@ public class CreateVPCCmdByAdminTest extends TestCase { @Mock public AccountService _accountService; private ResponseGenerator responseGenerator; + @Mock + public Object job; @InjectMocks - CreateVPCCmdByAdmin cmd = new CreateVPCCmdByAdmin(); + CreateVPCCmdByAdmin cmd; @Test public void testBgpPeerIds() { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmdTest.java index 415ee01ba16..c42a7882c54 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmdTest.java @@ -41,7 +41,10 @@ public class UpdateNetworkCmdTest { NetworkService networkService; @Mock public EntityManager _entityMgr; - private ResponseGenerator responseGenerator; + @Mock + private ResponseGenerator _responseGenerator; + @Mock + private Object job; @InjectMocks UpdateNetworkCmd cmd = new UpdateNetworkCmd(); @@ -176,15 +179,13 @@ public class UpdateNetworkCmdTest { ReflectionTestUtils.setField(cmd, "id", networkId); ReflectionTestUtils.setField(cmd, "publicMtu", publicmtu); Network network = Mockito.mock(Network.class); - responseGenerator = Mockito.mock(ResponseGenerator.class); NetworkResponse response = Mockito.mock(NetworkResponse.class); response.setPublicMtu(publicmtu); Mockito.when(networkService.getNetwork(networkId)).thenReturn(network); Mockito.when(networkService.updateGuestNetwork(cmd)).thenReturn(network); - cmd._responseGenerator = responseGenerator; - Mockito.when(responseGenerator.createNetworkResponse(ResponseObject.ResponseView.Restricted, network)).thenReturn(response); + Mockito.when(_responseGenerator.createNetworkResponse(ResponseObject.ResponseView.Restricted, network)).thenReturn(response); cmd.execute(); - Mockito.verify(responseGenerator).createNetworkResponse(Mockito.any(ResponseObject.ResponseView.class), Mockito.any(Network.class)); + Mockito.verify(_responseGenerator).createNetworkResponse(Mockito.any(ResponseObject.ResponseView.class), Mockito.any(Network.class)); NetworkResponse actualResponse = (NetworkResponse) cmd.getResponseObject(); Assert.assertEquals(response, actualResponse); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java index 2505c67e87d..18c03f8f4bb 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmdTest.java @@ -25,7 +25,6 @@ import com.cloud.network.vpc.Vpc; import com.cloud.network.vpc.VpcService; import com.cloud.user.AccountService; import com.cloud.utils.db.EntityManager; -import junit.framework.TestCase; import org.apache.cloudstack.api.ResponseGenerator; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.VpcResponse; @@ -39,7 +38,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; @RunWith(MockitoJUnitRunner.class) -public class CreateVPCCmdTest extends TestCase { +public class CreateVPCCmdTest { @Mock public VpcService _vpcService; @@ -47,9 +46,13 @@ public class CreateVPCCmdTest extends TestCase { public EntityManager _entityMgr; @Mock public AccountService _accountService; - private ResponseGenerator responseGenerator; + @Mock + private ResponseGenerator _responseGenerator; + @Mock + private Object job; + @InjectMocks - CreateVPCCmd cmd = new CreateVPCCmd(); + CreateVPCCmd cmd; @Test public void testGetAccountName() { @@ -185,11 +188,9 @@ public class CreateVPCCmdTest extends TestCase { VpcResponse response = Mockito.mock(VpcResponse.class); ReflectionTestUtils.setField(cmd, "id", 1L); - responseGenerator = Mockito.mock(ResponseGenerator.class); Mockito.doNothing().when(_vpcService).startVpc(cmd); Mockito.when(_entityMgr.findById(Mockito.eq(Vpc.class), Mockito.any(Long.class))).thenReturn(vpc); - cmd._responseGenerator = responseGenerator; - Mockito.when(responseGenerator.createVpcResponse(ResponseObject.ResponseView.Restricted, vpc)).thenReturn(response); + Mockito.when(_responseGenerator.createVpcResponse(ResponseObject.ResponseView.Restricted, vpc)).thenReturn(response); cmd.execute(); Mockito.verify(_vpcService, Mockito.times(1)).startVpc(cmd); } diff --git a/client/bindir/cloud-setup-management.in b/client/bindir/cloud-setup-management.in index 70e727b40b2..84c87ae2e44 100755 --- a/client/bindir/cloud-setup-management.in +++ b/client/bindir/cloud-setup-management.in @@ -16,13 +16,27 @@ # specific language governing permissions and limitations # under the License. +import os import sys +# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ---- +# ---- We do this so cloud_utils can be looked up in the following order: +# ---- 1) Sources directory +# ---- 2) waf configured PYTHONDIR +# ---- 3) System Python path +for pythonpath in ( + "@PYTHONDIR@", + os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"), + ): + if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath) +# ---- End snippet of code ---- + from cloudutils.syscfg import sysConfigFactory from cloudutils.utilities import initLoging, UnknownSystemException from cloudutils.cloudException import CloudRuntimeException, CloudInternalException from cloudutils.globalEnv import globalEnv from cloudutils.serviceConfigServer import cloudManagementConfig from optparse import OptionParser + if __name__ == '__main__': initLoging("@MSLOGDIR@/setupManagement.log") glbEnv = globalEnv() diff --git a/client/pom.xml b/client/pom.xml index e12e0395482..2b673d7750e 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -642,6 +642,11 @@ cloud-plugin-storage-object-ceph ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-object-cloudian + ${project.version} + org.apache.cloudstack cloud-plugin-storage-object-simulator diff --git a/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java new file mode 100644 index 00000000000..c19a061f736 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotAnswer.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class ConvertSnapshotAnswer extends Answer { + + private SnapshotObjectTO snapshotObjectTO; + + public ConvertSnapshotAnswer(SnapshotObjectTO snapshotObjectTO) { + super(null); + this.snapshotObjectTO = snapshotObjectTO; + } + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java new file mode 100644 index 00000000000..e456d4b6b12 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/ConvertSnapshotCommand.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class ConvertSnapshotCommand extends Command { + + public static final String TEMP_SNAPSHOT_NAME = "_temp"; + + SnapshotObjectTO snapshotObjectTO; + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } + + public ConvertSnapshotCommand(SnapshotObjectTO snapshotObjectTO) { + this.snapshotObjectTO = snapshotObjectTO; + } + + @Override + public boolean executeInSequence() { + return true; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/GetVmIpAddressCommand.java b/core/src/main/java/com/cloud/agent/api/GetVmIpAddressCommand.java index a9c7413b9f0..4efc0781f91 100644 --- a/core/src/main/java/com/cloud/agent/api/GetVmIpAddressCommand.java +++ b/core/src/main/java/com/cloud/agent/api/GetVmIpAddressCommand.java @@ -24,11 +24,13 @@ public class GetVmIpAddressCommand extends Command { String vmName; String vmNetworkCidr; boolean windows = false; + String macAddress; - public GetVmIpAddressCommand(String vmName, String vmNetworkCidr, boolean windows) { + public GetVmIpAddressCommand(String vmName, String vmNetworkCidr, boolean windows, String macAddress) { this.vmName = vmName; this.windows = windows; this.vmNetworkCidr = vmNetworkCidr; + this.macAddress = macAddress; } @Override @@ -47,4 +49,8 @@ public class GetVmIpAddressCommand extends Command { public String getVmNetworkCidr() { return vmNetworkCidr; } + + public String getMacAddress() { + return macAddress; + } } diff --git a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java index 3acdb9c351b..5ac4e9ae445 100644 --- a/core/src/main/java/com/cloud/agent/api/MigrateCommand.java +++ b/core/src/main/java/com/cloud/agent/api/MigrateCommand.java @@ -29,14 +29,14 @@ import com.cloud.agent.api.to.VirtualMachineTO; public class MigrateCommand extends Command { private String vmName; - private String destIp; + private String destinationIp; private Map migrateStorage; private boolean migrateStorageManaged; private boolean migrateNonSharedInc; private boolean autoConvergence; private String hostGuid; - private boolean isWindows; - private VirtualMachineTO vmTO; + private boolean windows; + private VirtualMachineTO virtualMachine; private boolean executeInSequence = false; private List migrateDiskInfoList = new ArrayList<>(); private Map dpdkInterfaceMapping = new HashMap<>(); @@ -64,11 +64,11 @@ public class MigrateCommand extends Command { protected MigrateCommand() { } - public MigrateCommand(String vmName, String destIp, boolean isWindows, VirtualMachineTO vmTO, boolean executeInSequence) { + public MigrateCommand(String vmName, String destinationIp, boolean windows, VirtualMachineTO virtualMachine, boolean executeInSequence) { this.vmName = vmName; - this.destIp = destIp; - this.isWindows = isWindows; - this.vmTO = vmTO; + this.destinationIp = destinationIp; + this.windows = windows; + this.virtualMachine = virtualMachine; this.executeInSequence = executeInSequence; } @@ -105,15 +105,15 @@ public class MigrateCommand extends Command { } public boolean isWindows() { - return isWindows; + return windows; } public VirtualMachineTO getVirtualMachine() { - return vmTO; + return virtualMachine; } public String getDestinationIp() { - return destIp; + return destinationIp; } public String getVmName() { @@ -233,4 +233,9 @@ public class MigrateCommand extends Command { this.isSourceDiskOnStorageFileSystem = isDiskOnFileSystemStorage; } } + + @Override + public boolean isReconcile() { + return true; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java index 552ffb85aaf..a3401d6faed 100644 --- a/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/ModifyStoragePoolAnswer.java @@ -46,6 +46,10 @@ public class ModifyStoragePoolAnswer extends Answer { templateInfo = tInfo; } + public ModifyStoragePoolAnswer(ModifyStoragePoolCommand cmd, boolean success, String details) { + super(cmd, success, details); + } + public void setPoolInfo(StoragePoolInfo poolInfo) { this.poolInfo = poolInfo; } diff --git a/core/src/main/java/com/cloud/agent/api/PingAnswer.java b/core/src/main/java/com/cloud/agent/api/PingAnswer.java index 6353b121583..22eb4443f8a 100644 --- a/core/src/main/java/com/cloud/agent/api/PingAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/PingAnswer.java @@ -19,18 +19,25 @@ package com.cloud.agent.api; +import java.util.ArrayList; +import java.util.List; + public class PingAnswer extends Answer { private PingCommand _command = null; private boolean sendStartup = false; + private List avoidMsList; + + private List reconcileCommands = new ArrayList<>(); protected PingAnswer() { } - public PingAnswer(PingCommand cmd, boolean sendStartup) { + public PingAnswer(PingCommand cmd, List avoidMsList, boolean sendStartup) { super(cmd); _command = cmd; this.sendStartup = sendStartup; + this.avoidMsList = avoidMsList; } public PingCommand getCommand() { @@ -44,4 +51,20 @@ public class PingAnswer extends Answer { public void setSendStartup(boolean sendStartup) { this.sendStartup = sendStartup; } + + public List getReconcileCommands() { + return reconcileCommands; + } + + public void setReconcileCommands(List reconcileCommands) { + this.reconcileCommands = reconcileCommands; + } + + public void addReconcileCommand(String reconcileCommand) { + this.reconcileCommands.add(reconcileCommand); + } + + public List getAvoidMsList() { + return avoidMsList; + } } diff --git a/core/src/main/java/com/cloud/agent/api/PingCommand.java b/core/src/main/java/com/cloud/agent/api/PingCommand.java index 4192fc2e747..4ae64da89c3 100644 --- a/core/src/main/java/com/cloud/agent/api/PingCommand.java +++ b/core/src/main/java/com/cloud/agent/api/PingCommand.java @@ -20,11 +20,14 @@ package com.cloud.agent.api; import com.cloud.host.Host; +import org.apache.cloudstack.command.CommandInfo; public class PingCommand extends Command { Host.Type hostType; long hostId; boolean outOfBand; + @LogLevel(LogLevel.Log4jLevel.Trace) + private CommandInfo[] commandInfos = new CommandInfo[] {}; protected PingCommand() { } @@ -78,4 +81,12 @@ public class PingCommand extends Command { result = 31 * result + (int) (hostId ^ (hostId >>> 32)); return result; } + + public CommandInfo[] getCommandInfos() { + return commandInfos; + } + + public void setCommandInfos(CommandInfo[] commandInfos) { + this.commandInfos = commandInfos; + } } diff --git a/core/src/main/java/com/cloud/agent/api/ReadyCommand.java b/core/src/main/java/com/cloud/agent/api/ReadyCommand.java index e2d974e3878..49768297ad5 100644 --- a/core/src/main/java/com/cloud/agent/api/ReadyCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ReadyCommand.java @@ -35,6 +35,7 @@ public class ReadyCommand extends Command { private String hostUuid; private String hostName; private List msHostList; + private List avoidMsHostList; private String lbAlgorithm; private Long lbCheckInterval; private Boolean enableHumanReadableSizes; @@ -90,6 +91,14 @@ public class ReadyCommand extends Command { this.msHostList = msHostList; } + public List getAvoidMsHostList() { + return avoidMsHostList; + } + + public void setAvoidMsHostList(List msHostList) { + this.avoidMsHostList = avoidMsHostList; + } + public String getLbAlgorithm() { return lbAlgorithm; } diff --git a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java b/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java similarity index 55% rename from plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java rename to core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java index 81c58ef27ba..154b611be98 100644 --- a/plugins/hypervisors/vmware/src/main/java/org/apache/cloudstack/api/command/admin/zone/VmwareRequestResponse.java +++ b/core/src/main/java/com/cloud/agent/api/RecreateCheckpointsCommand.java @@ -1,3 +1,4 @@ +// // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -14,25 +15,35 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +// -package org.apache.cloudstack.api.command.admin.zone; +package com.cloud.agent.api; -import com.cloud.serializer.Param; -import com.google.gson.annotations.SerializedName; -import org.apache.cloudstack.api.ApiConstants; -import org.apache.cloudstack.api.ResponseObject; -import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.storage.to.VolumeObjectTO; -public class VmwareRequestResponse extends ListResponse { - @SerializedName(ApiConstants.TOKEN) - @Param(description = "The Vmware API token to use for retrieving further responses with") - private String token; +import java.util.List; - public String getToken() { - return token; +public class RecreateCheckpointsCommand extends Command { + + private List volumes; + + private String vmName; + + public RecreateCheckpointsCommand(List volumes, String vmName) { + this.volumes = volumes; + this.vmName = vmName; } - public void setToken(String token) { - this.token = token; + public List getDisks() { + return volumes; + } + + public String getVmName() { + return vmName; + } + + @Override + public boolean executeInSequence() { + return true; } } diff --git a/core/src/main/java/com/cloud/agent/api/RemoveBitmapCommand.java b/core/src/main/java/com/cloud/agent/api/RemoveBitmapCommand.java new file mode 100644 index 00000000000..90b62e3f17b --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/RemoveBitmapCommand.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +import org.apache.cloudstack.storage.to.SnapshotObjectTO; + +public class RemoveBitmapCommand extends Command { + + private SnapshotObjectTO snapshotObjectTO; + + private boolean isVmRunning; + + public RemoveBitmapCommand(SnapshotObjectTO snapshotObjectTO, boolean isVmRunning) { + this.snapshotObjectTO = snapshotObjectTO; + this.isVmRunning = isVmRunning; + } + + @Override + public boolean executeInSequence() { + return true; + } + + public SnapshotObjectTO getSnapshotObjectTO() { + return snapshotObjectTO; + } + + public boolean isVmRunning() { + return isVmRunning; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java b/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java index bebd30ca519..b98ce6fa345 100644 --- a/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java +++ b/core/src/main/java/com/cloud/agent/api/UnprepareStorageClientCommand.java @@ -19,18 +19,22 @@ package com.cloud.agent.api; +import java.util.Map; + import com.cloud.storage.Storage.StoragePoolType; public class UnprepareStorageClientCommand extends Command { private StoragePoolType poolType; private String poolUuid; + private Map details; public UnprepareStorageClientCommand() { } - public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid) { + public UnprepareStorageClientCommand(StoragePoolType poolType, String poolUuid, Map details) { this.poolType = poolType; this.poolUuid = poolUuid; + this.details = details; } @Override @@ -45,4 +49,8 @@ public class UnprepareStorageClientCommand extends Command { public String getPoolUuid() { return poolUuid; } + + public Map getDetails() { + return details; + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java b/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java index 9fed0f913e1..70375c30a1b 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/MigrateVolumeCommand.java @@ -145,4 +145,9 @@ public class MigrateVolumeCommand extends Command { } public String getChainInfo() { return chainInfo; } + + @Override + public boolean isReconcile() { + return true; + } } diff --git a/core/src/main/java/com/cloud/resource/ServerResource.java b/core/src/main/java/com/cloud/resource/ServerResource.java index 845ac8a48fa..23d200942a2 100644 --- a/core/src/main/java/com/cloud/resource/ServerResource.java +++ b/core/src/main/java/com/cloud/resource/ServerResource.java @@ -22,6 +22,7 @@ package com.cloud.resource; import com.cloud.agent.IAgentControl; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; +import com.cloud.agent.api.PingAnswer; import com.cloud.agent.api.PingCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.host.Host; @@ -90,4 +91,5 @@ public interface ServerResource extends Manager { return false; } + default void processPingAnswer(PingAnswer answer) {}; } diff --git a/core/src/main/java/com/cloud/serializer/GsonHelper.java b/core/src/main/java/com/cloud/serializer/GsonHelper.java index 2d2cecf2618..7de98c08b7e 100644 --- a/core/src/main/java/com/cloud/serializer/GsonHelper.java +++ b/core/src/main/java/com/cloud/serializer/GsonHelper.java @@ -62,7 +62,7 @@ public class GsonHelper { LOGGER.info("Default Builder inited."); } - static Gson setDefaultGsonConfig(GsonBuilder builder) { + public static Gson setDefaultGsonConfig(GsonBuilder builder) { builder.setVersion(1.5); InterfaceTypeAdaptor dsAdaptor = new InterfaceTypeAdaptor(); builder.registerTypeAdapter(DataStoreTO.class, dsAdaptor); diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 7d8225462ca..58885210120 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -142,7 +142,7 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma } return new CreateObjectAnswer("not supported type"); } catch (Exception e) { - logger.debug("Failed to create object: " + data.getObjectType() + ": " + e.toString()); + logger.error("Failed to create object [{}] due to [{}].", data.getObjectType(), e.getMessage(), e); return new CreateObjectAnswer(e.toString()); } } diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 9b126846827..4c056f256cf 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -27,6 +27,7 @@ import java.io.InputStream; import java.io.RandomAccessFile; import java.net.URI; import java.net.URISyntaxException; +import java.util.Arrays; import java.util.Date; import java.util.List; @@ -80,6 +81,18 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te private ResourceType resourceType = ResourceType.TEMPLATE; private final HttpMethodRetryHandler myretryhandler; private boolean followRedirects = false; + private boolean isChunkedTransfer; + + protected static final List CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE = Arrays.asList( + "x-goog-stored-content-length", + "x-goog-meta-size", + "x-amz-meta-size", + "x-amz-meta-content-length", + "x-object-meta-size", + "x-original-content-length", + "x-oss-meta-content-length", + "x-file-size"); + private static final long MIN_FORMAT_VERIFICATION_SIZE = 1024 * 1024; public HttpTemplateDownloader(StorageLayer storageLayer, String downloadUrl, String toDir, DownloadCompleteCallback callback, long maxTemplateSizeInBytes, String user, String password, Proxy proxy, ResourceType resourceType) { @@ -205,13 +218,11 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te RandomAccessFile out = new RandomAccessFile(file, "rw"); ) { out.seek(localFileSize); - - logger.info("Starting download from " + downloadUrl + " to " + toFile + " remoteSize=" + toHumanReadableSize(remoteSize) + " , max size=" + toHumanReadableSize(maxTemplateSizeInBytes)); - - if (copyBytes(file, in, out)) return 0; - + logger.info("Starting download from {} to {} remoteSize={} , max size={}",downloadUrl, toFile, + toHumanReadableSize(remoteSize), toHumanReadableSize(maxTemplateSizeInBytes)); + boolean eof = copyBytes(file, in, out); Date finish = new Date(); - checkDowloadCompletion(); + checkDownloadCompletion(eof); downloadTime += finish.getTime() - start.getTime(); } finally { /* in.close() and out.close() */ } return totalBytes; @@ -237,28 +248,32 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te } private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throws IOException { - int bytes; - byte[] block = new byte[CHUNK_SIZE]; + byte[] buffer = new byte[CHUNK_SIZE]; long offset = 0; - boolean done = false; VerifyFormat verifyFormat = new VerifyFormat(file); status = Status.IN_PROGRESS; - while (!done && status != Status.ABORTED && offset <= remoteSize) { - if ((bytes = in.read(block, 0, CHUNK_SIZE)) > -1) { - offset = writeBlock(bytes, out, block, offset); - if (!ResourceType.SNAPSHOT.equals(resourceType) && - !verifyFormat.isVerifiedFormat() && - (offset >= 1048576 || offset >= remoteSize)) { //let's check format after we get 1MB or full file - verifyFormat.invoke(); - } - } else { - done = true; + while (status != Status.ABORTED) { + int bytesRead = in.read(buffer, 0, CHUNK_SIZE); + if (bytesRead == -1) { + logger.debug("Reached EOF on input stream"); + break; + } + offset = writeBlock(bytesRead, out, buffer, offset); + if (!ResourceType.SNAPSHOT.equals(resourceType) + && !verifyFormat.isVerifiedFormat() + && (offset >= MIN_FORMAT_VERIFICATION_SIZE || offset >= remoteSize)) { + verifyFormat.invoke(); + } + if (offset >= remoteSize) { + logger.debug("Reached expected remote size limit: {} bytes", remoteSize); + break; } } out.getFD().sync(); - return false; + return !Status.ABORTED.equals(status); } + private long writeBlock(int bytes, RandomAccessFile out, byte[] block, long offset) throws IOException { out.write(block, 0, bytes); offset += bytes; @@ -267,11 +282,13 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te return offset; } - private void checkDowloadCompletion() { + private void checkDownloadCompletion(boolean eof) { String downloaded = "(incomplete download)"; - if (totalBytes >= remoteSize) { + if (eof && ((totalBytes >= remoteSize) || (isChunkedTransfer && remoteSize == maxTemplateSizeInBytes))) { status = Status.DOWNLOAD_FINISHED; - downloaded = "(download complete remote=" + toHumanReadableSize(remoteSize) + " bytes)"; + downloaded = "(download complete remote=" + + (remoteSize == maxTemplateSizeInBytes ? toHumanReadableSize(remoteSize) : "unknown") + + " bytes)"; } errorString = "Downloaded " + toHumanReadableSize(totalBytes) + " bytes " + downloaded; } @@ -293,18 +310,42 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te } } + protected long getRemoteSizeForChunkedTransfer() { + for (String headerKey : CUSTOM_HEADERS_FOR_CHUNKED_TRANSFER_SIZE) { + Header header = request.getResponseHeader(headerKey); + if (header == null) { + continue; + } + try { + return Long.parseLong(header.getValue()); + } catch (NumberFormatException ignored) {} + } + Header contentRangeHeader = request.getResponseHeader("Content-Range"); + if (contentRangeHeader != null) { + String contentRange = contentRangeHeader.getValue(); + if (contentRange != null && contentRange.contains("/")) { + String totalSize = contentRange.substring(contentRange.indexOf('/') + 1).trim(); + return Long.parseLong(totalSize); + } + } + return 0; + } + private boolean tryAndGetRemoteSize() { Header contentLengthHeader = request.getResponseHeader("content-length"); - boolean chunked = false; + isChunkedTransfer = false; long reportedRemoteSize = 0; if (contentLengthHeader == null) { Header chunkedHeader = request.getResponseHeader("Transfer-Encoding"); - if (chunkedHeader == null || !"chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + if (chunkedHeader != null && "chunked".equalsIgnoreCase(chunkedHeader.getValue())) { + isChunkedTransfer = true; + reportedRemoteSize = getRemoteSizeForChunkedTransfer(); + logger.debug("{} is using chunked transfer encoding, possible remote size: {}", downloadUrl, + reportedRemoteSize); + } else { status = Status.UNRECOVERABLE_ERROR; errorString = " Failed to receive length of download "; return false; - } else if ("chunked".equalsIgnoreCase(chunkedHeader.getValue())) { - chunked = true; } } else { reportedRemoteSize = Long.parseLong(contentLengthHeader.getValue()); @@ -316,9 +357,11 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te return false; } } - if (remoteSize == 0) { remoteSize = reportedRemoteSize; + if (remoteSize != 0) { + logger.debug("Remote size for {} found to be {}", downloadUrl, toHumanReadableSize(remoteSize)); + } } return true; } diff --git a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java index d6622bed73e..349997da1b7 100644 --- a/core/src/main/java/com/cloud/storage/template/TemplateConstants.java +++ b/core/src/main/java/com/cloud/storage/template/TemplateConstants.java @@ -24,7 +24,7 @@ public final class TemplateConstants { public static final String DEFAULT_SNAPSHOT_ROOT_DIR = "snapshots"; public static final String DEFAULT_VOLUME_ROOT_DIR = "volumes"; public static final String DEFAULT_TMPLT_FIRST_LEVEL_DIR = "tmpl/"; - + public static final String DEFAULT_CHECKPOINT_ROOT_DIR = "checkpoints"; public static final String DEFAULT_SYSTEM_VM_TEMPLATE_PATH = "template/tmpl/1/"; public static final int DEFAULT_TMPLT_COPY_PORT = 80; diff --git a/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java b/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java index 50cf956c9e7..32f436434c1 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/lb/SetupMSListCommand.java @@ -26,12 +26,14 @@ import com.cloud.agent.api.Command; public class SetupMSListCommand extends Command { private List msList; + private List avoidMsList; private String lbAlgorithm; private Long lbCheckInterval; - public SetupMSListCommand(final List msList, final String lbAlgorithm, final Long lbCheckInterval) { + public SetupMSListCommand(final List msList, final List avoidMsList, final String lbAlgorithm, final Long lbCheckInterval) { super(); this.msList = msList; + this.avoidMsList = avoidMsList; this.lbAlgorithm = lbAlgorithm; this.lbCheckInterval = lbCheckInterval; } @@ -40,6 +42,10 @@ public class SetupMSListCommand extends Command { return msList; } + public List getAvoidMsList() { + return avoidMsList; + } + public String getLbAlgorithm() { return lbAlgorithm; } diff --git a/core/src/main/java/org/apache/cloudstack/command/CommandInfo.java b/core/src/main/java/org/apache/cloudstack/command/CommandInfo.java new file mode 100644 index 00000000000..b9bb702e345 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/CommandInfo.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Command; +import com.cloud.serializer.GsonHelper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +import java.util.Date; + +public class CommandInfo { + public static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSSZ"; + public static final Gson GSON = GsonHelper.setDefaultGsonConfig(new GsonBuilder().setDateFormat(DATE_FORMAT)); + + long requestSeq; + Command.State state; + Date startTime; + Date updateTime; + String commandName; + String command; + int timeout; + String answerName; + String answer; + + public CommandInfo() { + } + + public CommandInfo(long requestSeq, Command command, Command.State state) { + this.requestSeq = requestSeq; + this.state = state; + this.startTime = this.updateTime = new Date(); + this.commandName = command.getClass().getName(); + this.command = GSON.toJson(command); + this.timeout = command.getWait(); + } + + public long getRequestSeq() { + return requestSeq; + } + + public void setRequestSeq(long requestSeq) { + this.requestSeq = requestSeq; + } + + public Command.State getState() { + return state; + } + + public void setState(Command.State state) { + this.state = state; + this.updateTime = new Date(); + } + + public Date getStartTime() { + return startTime; + } + + public void setStartTime(Date startTime) { + this.startTime = startTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getCommandName() { + return commandName; + } + + public void setCommandName(String commandName) { + this.commandName = commandName; + } + + public String getCommand() { + return command; + } + + public void setCommand(String command) { + this.command = command; + } + + public int getTimeout() { + return timeout; + } + + public void setTimeout(int timeout) { + this.timeout = timeout; + } + + public String getAnswerName() { + return answerName; + } + + public void setAnswerName(String answerName) { + this.answerName = answerName; + } + + public String getAnswer() { + return answer; + } + + public void setAnswer(String answer) { + this.answer = answer; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java new file mode 100644 index 00000000000..e8d27e1fa71 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileAnswer.java @@ -0,0 +1,45 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Answer; +import org.apache.cloudstack.api.ApiCommandResourceType; + +public class ReconcileAnswer extends Answer { + + ApiCommandResourceType resourceType; + Long resourceId; + + public ApiCommandResourceType getResourceType() { + return resourceType; + } + + public void setResourceType(ApiCommandResourceType resourceType) { + this.resourceType = resourceType; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } +} diff --git a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java similarity index 63% rename from vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java rename to core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java index 828eed4622d..262aefb30ac 100644 --- a/vmware-base/src/main/java/com/cloud/hypervisor/vmware/util/VmwareClientException.java +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommand.java @@ -14,20 +14,20 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.hypervisor.vmware.util; -import com.cloud.exception.CloudException; +package org.apache.cloudstack.command; -public class VmwareClientException extends CloudException { - public VmwareClientException(String message, Throwable cause) { - super(message, cause); +import com.cloud.agent.api.Command; + +public class ReconcileCommand extends Command { + + @Override + public boolean executeInSequence() { + return false; } - public VmwareClientException(String msg) { - super(msg); - } - // TODO embed vmware classes in this one for use downstream - public VmwareClientException(String msg, Exception embedded) { - super(msg, embedded); + @Override + public int getWait() { + return 30; // timeout is 30 seconds } } diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java new file mode 100644 index 00000000000..8acc02a730f --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCommandUtils.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.JsonSyntaxException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.SimpleDateFormat; + +public class ReconcileCommandUtils { + + protected static final Logger LOGGER = LogManager.getLogger(ReconcileCommandUtils.class.getName()); + + public static void createLogFileForCommand(final String logPath, final Command cmd) { + updateLogFileForCommand(logPath, cmd, Command.State.CREATED); + } + + public static void updateLogFileForCommand(final String logPath, final Command cmd, final Command.State state) { + if (cmd.isReconcile()) { + String logFileName = getLogFileNameForCommand(logPath, cmd); + LOGGER.debug(String.format("Updating log file %s with %s state", logFileName, state)); + File logFile = new File(logFileName); + CommandInfo commandInfo = null; + if (logFile.exists()) { + commandInfo = readLogFileForCommand(logFileName); + logFile.delete(); + } + if (commandInfo == null) { + commandInfo = new CommandInfo(cmd.getRequestSequence(), cmd, state); + } else { + commandInfo.setState(state); + } + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); + writer.write(CommandInfo.GSON.toJson(commandInfo)); + writer.close(); + } catch (IOException e) { + LOGGER.error(String.format("Failed to write log file %s", logFile)); + } + } + } + + public static void updateLogFileForCommand(final String logFullPath, final Command.State state) { + File logFile = new File(logFullPath); + LOGGER.debug(String.format("Updating log file %s with %s state", logFile.getName(), state)); + if (!logFile.exists()) { + return; + } + CommandInfo commandInfo = readLogFileForCommand(logFullPath); + if (commandInfo != null) { + commandInfo.setState(state); + } + logFile.delete(); + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); + writer.write(CommandInfo.GSON.toJson(commandInfo)); + writer.close(); + } catch (IOException e) { + LOGGER.error(String.format("Failed to write log file %s", logFile)); + } + } + + public static void deleteLogFileForCommand(final String logPath, final Command cmd) { + if (cmd.isReconcile()) { + File logFile = new File(getLogFileNameForCommand(logPath, cmd)); + LOGGER.debug(String.format("Removing log file %s", logFile.getName())); + if (logFile.exists()) { + logFile.delete(); + } + } + } + + public static void deleteLogFile(final String logFullPath) { + File logFile = new File(logFullPath); + LOGGER.debug(String.format("Removing log file %s ", logFile.getName())); + if (logFile.exists()) { + logFile.delete(); + } + } + + public static String getLogFileNameForCommand(final String logPath, final Command cmd) { + return String.format("%s/%s-%s.json", logPath, cmd.getRequestSequence(), cmd); + } + + public static CommandInfo readLogFileForCommand(final String logFullPath) { + try { + ObjectMapper objectMapper = new ObjectMapper(); + SimpleDateFormat df = new SimpleDateFormat(CommandInfo.DATE_FORMAT); + objectMapper.setDateFormat(df); + return objectMapper.readValue(new File(logFullPath), CommandInfo.class); + } catch (IOException e) { + LOGGER.error(String.format("Failed to read log file %s: %s", logFullPath, e.getMessage())); + return null; + } + } + + public static Command parseCommandInfo(final CommandInfo commandInfo) { + if (commandInfo.getCommandName() == null || commandInfo.getCommand() == null) { + return null; + } + return parseCommandInfo(commandInfo.getCommandName(), commandInfo.getCommand()); + } + + public static Command parseCommandInfo(final String commandName, final String commandInfo) { + Object parsedObject = null; + try { + Class commandClazz = Class.forName(commandName); + parsedObject = CommandInfo.GSON.fromJson(commandInfo, commandClazz); + } catch (ClassNotFoundException | JsonSyntaxException e) { + LOGGER.error(String.format("Failed to parse command from CommandInfo %s due to %s", commandInfo, e.getMessage())); + } + if (parsedObject != null) { + return (Command) parsedObject; + } + return null; + } + + public static Answer parseAnswerFromCommandInfo(final CommandInfo commandInfo) { + if (commandInfo.getAnswerName() == null || commandInfo.getAnswer() == null) { + return null; + } + return parseAnswerFromAnswerInfo(commandInfo.getAnswerName(), commandInfo.getAnswer()); + } + + public static Answer parseAnswerFromAnswerInfo(final String answerName, final String answerInfo) { + Object parsedObject = null; + try { + Class commandClazz = Class.forName(answerName); + parsedObject = CommandInfo.GSON.fromJson(answerInfo, commandClazz); + } catch (ClassNotFoundException | JsonSyntaxException e) { + LOGGER.error(String.format("Failed to parse answer from answerInfo %s due to %s", answerInfo, e.getMessage())); + } + if (parsedObject != null) { + return (Answer) parsedObject; + } + return null; + } + + public static void updateLogFileWithAnswerForCommand(final String logPath, final Command cmd, final Answer answer) { + if (cmd.isReconcile()) { + String logFileName = getLogFileNameForCommand(logPath, cmd); + LOGGER.debug(String.format("Updating log file %s with answer %s", logFileName, answer)); + File logFile = new File(logFileName); + if (!logFile.exists()) { + return; + } + CommandInfo commandInfo = readLogFileForCommand(logFile.getAbsolutePath()); + if (commandInfo == null) { + return; + } + if (Command.State.STARTED.equals(commandInfo.getState())) { + if (answer.getResult()) { + commandInfo.setState(Command.State.COMPLETED); + } else { + commandInfo.setState(Command.State.FAILED); + } + } + commandInfo.setAnswerName(answer.toString()); + commandInfo.setAnswer(CommandInfo.GSON.toJson(answer)); + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(logFile)); + writer.write(CommandInfo.GSON.toJson(commandInfo)); + writer.close(); + } catch (IOException e) { + LOGGER.error(String.format("Failed to write log file %s", logFile)); + } + } + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java new file mode 100644 index 00000000000..82a24fa7fa5 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyAnswer.java @@ -0,0 +1,56 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +public class ReconcileCopyAnswer extends ReconcileVolumeAnswer { + + boolean isSkipped = false; + String reason; + + public ReconcileCopyAnswer(boolean isSkipped, String reason) { + super(); + this.isSkipped = isSkipped; + this.reason = reason; + } + + public ReconcileCopyAnswer(boolean isSkipped, boolean result, String reason) { + super(); + this.isSkipped = isSkipped; + this.result = result; + this.reason = reason; + } + + public ReconcileCopyAnswer(VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination) { + this.isSkipped = false; + this.result = true; + this.volumeOnSource = volumeOnSource; + this.volumeOnDestination = volumeOnDestination; + } + + public boolean isSkipped() { + return isSkipped; + } + + public String getReason() { + return reason; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java new file mode 100644 index 00000000000..36a678833d0 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileCopyCommand.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.to.DataTO; + +import java.util.Map; + +public class ReconcileCopyCommand extends ReconcileCommand { + + DataTO srcData; + DataTO destData; + Map option; // details of source volume + Map option2; // details of destination volume + + public ReconcileCopyCommand(DataTO srcData, DataTO destData, Map option, Map option2) { + this.srcData = srcData; + this.destData = destData; + this.option = option; + this.option2 = option2; + } + + public DataTO getSrcData() { + return srcData; + } + + public DataTO getDestData() { + return destData; + } + + public Map getOption() { + return option; + } + + public Map getOption2() { + return option2; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java new file mode 100644 index 00000000000..6313267c7e4 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateAnswer.java @@ -0,0 +1,68 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import com.cloud.vm.VirtualMachine; + +import java.util.List; + +public class ReconcileMigrateAnswer extends ReconcileAnswer { + + Long hostId; + String vmName; + VirtualMachine.State vmState; + List vmDisks; + + public ReconcileMigrateAnswer() { + } + + public ReconcileMigrateAnswer(String vmName, VirtualMachine.State vmState) { + this.vmName = vmName; + this.vmState = vmState; + } + + public Long getHostId() { + return hostId; + } + + public void setHostId(Long hostId) { + this.hostId = hostId; + } + + public String getVmName() { + return vmName; + } + + public VirtualMachine.State getVmState() { + return vmState; + } + + public void setVmState(VirtualMachine.State vmState) { + this.vmState = vmState; + } + + public List getVmDisks() { + return vmDisks; + } + + public void setVmDisks(List vmDisks) { + this.vmDisks = vmDisks; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java new file mode 100644 index 00000000000..50e1c68a65f --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateCommand.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +public class ReconcileMigrateCommand extends ReconcileCommand { + + String vmName; + + public ReconcileMigrateCommand(String vmName) { + this.vmName = vmName; + } + + public String getVmName() { + return vmName; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java new file mode 100644 index 00000000000..ebbd913f971 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeAnswer.java @@ -0,0 +1,50 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +import java.util.List; + +public class ReconcileMigrateVolumeAnswer extends ReconcileVolumeAnswer { + + String vmName; + List vmDiskPaths; + + public ReconcileMigrateVolumeAnswer(VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination) { + super(volumeOnSource, volumeOnDestination); + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } + + public List getVmDiskPaths() { + return vmDiskPaths; + } + + public void setVmDiskPaths(List vmDiskPaths) { + this.vmDiskPaths = vmDiskPaths; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java new file mode 100644 index 00000000000..e3e75249406 --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileMigrateVolumeCommand.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.command; + +import com.cloud.agent.api.to.DataTO; + +public class ReconcileMigrateVolumeCommand extends ReconcileCommand { + + DataTO srcData; + DataTO destData; + String vmName; + + public ReconcileMigrateVolumeCommand(DataTO srcData, DataTO destData) { + this.srcData = srcData; + this.destData = destData; + } + + public DataTO getSrcData() { + return srcData; + } + + public DataTO getDestData() { + return destData; + } + + public String getVmName() { + return vmName; + } + + public void setVmName(String vmName) { + this.vmName = vmName; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java b/core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java new file mode 100644 index 00000000000..da10bf23aee --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/command/ReconcileVolumeAnswer.java @@ -0,0 +1,46 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.command; + +import org.apache.cloudstack.storage.volume.VolumeOnStorageTO; + +public class ReconcileVolumeAnswer extends ReconcileAnswer { + + // (1) null object: volume is not available. For example the source is secondary storage + // (2) otherwise, if volumeOnSource.getPath() is null, the volume cannot be found on primary storage pool + VolumeOnStorageTO volumeOnSource; + VolumeOnStorageTO volumeOnDestination; + + public ReconcileVolumeAnswer() { + } + + public ReconcileVolumeAnswer(VolumeOnStorageTO volumeOnSource, VolumeOnStorageTO volumeOnDestination) { + this.volumeOnSource = volumeOnSource; + this.volumeOnDestination = volumeOnDestination; + } + + public VolumeOnStorageTO getVolumeOnSource() { + return volumeOnSource; + } + + public VolumeOnStorageTO getVolumeOnDestination() { + return volumeOnDestination; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java index aac082a0133..36550420909 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/CopyCommand.java @@ -93,4 +93,9 @@ public class CopyCommand extends StorageSubSystemCommand { public void setExecuteInSequence(final boolean inSeq) { executeInSequence = inSeq; } + + @Override + public boolean isReconcile() { + return true; + } } diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java index 76b93909b8c..0c3bb99e75c 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/SnapshotObjectTO.java @@ -35,12 +35,16 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { private VolumeObjectTO volume; private String parentSnapshotPath; private DataStoreTO dataStore; + private DataStoreTO imageStore; + private boolean kvmIncrementalSnapshot = false; private String vmName; private String name; private HypervisorType hypervisorType; private long id; private boolean quiescevm; private String[] parents; + private DataStoreTO parentStore; + private String checkpointPath; private Long physicalSize = (long) 0; private long accountId; @@ -49,6 +53,11 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { } + @Override + public DataObjectType getObjectType() { + return DataObjectType.SNAPSHOT; + } + public SnapshotObjectTO(SnapshotInfo snapshot) { this.path = snapshot.getPath(); this.setId(snapshot.getId()); @@ -59,27 +68,28 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { this.setVmName(vol.getAttachedVmName()); } - SnapshotInfo parentSnapshot = snapshot.getParent(); - ArrayList parentsArry = new ArrayList(); - if (parentSnapshot != null) { - this.parentSnapshotPath = parentSnapshot.getPath(); - while(parentSnapshot != null) { - parentsArry.add(parentSnapshot.getPath()); - parentSnapshot = parentSnapshot.getParent(); - } - parents = parentsArry.toArray(new String[parentsArry.size()]); - ArrayUtils.reverse(parents); - } - this.dataStore = snapshot.getDataStore().getTO(); this.setName(snapshot.getName()); this.hypervisorType = snapshot.getHypervisorType(); this.quiescevm = false; - } - @Override - public DataObjectType getObjectType() { - return DataObjectType.SNAPSHOT; + this.checkpointPath = snapshot.getCheckpointPath(); + this.kvmIncrementalSnapshot = snapshot.isKvmIncrementalSnapshot(); + + SnapshotInfo parentSnapshot = snapshot.getParent(); + + if (parentSnapshot == null || (HypervisorType.KVM.equals(snapshot.getHypervisorType()) && !parentSnapshot.isKvmIncrementalSnapshot())) { + return; + } + + ArrayList parentsArray = new ArrayList<>(); + this.parentSnapshotPath = parentSnapshot.getPath(); + while (parentSnapshot != null) { + parentsArray.add(parentSnapshot.getPath()); + parentSnapshot = parentSnapshot.getParent(); + } + parents = parentsArray.toArray(new String[parentsArray.size()]); + ArrayUtils.reverse(parents); } @Override @@ -91,6 +101,30 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { this.dataStore = store; } + public DataStoreTO getImageStore() { + return imageStore; + } + + public void setImageStore(DataStoreTO imageStore) { + this.imageStore = imageStore; + } + + public boolean isKvmIncrementalSnapshot() { + return kvmIncrementalSnapshot; + } + + public void setKvmIncrementalSnapshot(boolean kvmIncrementalSnapshot) { + this.kvmIncrementalSnapshot = kvmIncrementalSnapshot; + } + + public String getCheckpointPath() { + return checkpointPath; + } + + public void setCheckpointPath(String checkpointPath) { + this.checkpointPath = checkpointPath; + } + @Override public String getPath() { return this.path; @@ -178,6 +212,14 @@ public class SnapshotObjectTO extends DownloadableObjectTO implements DataTO { this.accountId = accountId; } + public DataStoreTO getParentStore() { + return parentStore; + } + + public void setParentStore(DataStoreTO parentStore) { + this.parentStore = parentStore; + } + @Override public String toString() { return new StringBuilder("SnapshotTO[datastore=").append(dataStore).append("|volume=").append(volume).append("|path").append(path).append("]").toString(); diff --git a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java index 4d1d0bf9097..92f9ee497a4 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java +++ b/core/src/main/java/org/apache/cloudstack/storage/to/VolumeObjectTO.java @@ -33,6 +33,8 @@ import com.cloud.storage.Volume; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import java.util.Arrays; +import java.util.List; +import java.util.Set; public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { private String uuid; @@ -76,6 +78,8 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { @LogLevel(LogLevel.Log4jLevel.Off) private byte[] passphrase; private String encryptFormat; + private List checkpointPaths; + private Set checkpointImageStoreUrls; public VolumeObjectTO() { @@ -122,6 +126,8 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { this.passphrase = volume.getPassphrase(); this.encryptFormat = volume.getEncryptFormat(); this.followRedirects = volume.isFollowRedirects(); + this.checkpointPaths = volume.getCheckpointPaths(); + this.checkpointImageStoreUrls = volume.getCheckpointImageStoreUrls(); } public String getUuid() { @@ -397,4 +403,21 @@ public class VolumeObjectTO extends DownloadableObjectTO implements DataTO { public boolean requiresEncryption() { return passphrase != null && passphrase.length > 0; } + + + public List getCheckpointPaths() { + return checkpointPaths; + } + + public void setCheckpointPaths(List checkpointPaths) { + this.checkpointPaths = checkpointPaths; + } + + public Set getCheckpointImageStoreUrls() { + return checkpointImageStoreUrls; + } + + public void setCheckpointImageStoreUrls(Set checkpointImageStoreUrls) { + this.checkpointImageStoreUrls = checkpointImageStoreUrls; + } } diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java index 3f68422d502..f021009cec0 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckGuestOsMappingCommandTest.java @@ -25,8 +25,6 @@ import static org.junit.Assert.assertFalse; import com.cloud.agent.api.CheckGuestOsMappingCommand; import org.junit.Test; -import com.cloud.agent.api.AgentControlCommand; - public class CheckGuestOsMappingCommandTest { @Test diff --git a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckOnHostCommandTest.java b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckOnHostCommandTest.java index be7563be045..a696049608e 100644 --- a/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckOnHostCommandTest.java +++ b/core/src/test/java/org/apache/cloudstack/api/agent/test/CheckOnHostCommandTest.java @@ -284,6 +284,11 @@ public class CheckOnHostCommandTest { public CPU.CPUArch getArch() { return CPU.CPUArch.amd64; } + + @Override + public String getStorageAccessGroups() { + return null; + } }; CheckOnHostCommand cohc = new CheckOnHostCommand(host); diff --git a/core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java b/core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java new file mode 100644 index 00000000000..f69ada58e1a --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/command/ReconcileCommandUtilsTest.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.command; + +import com.cloud.agent.api.Command; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.to.VirtualMachineTO; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; + +public class ReconcileCommandUtilsTest { + + final static String COMMANDS_LOG_PATH = "/tmp"; + + @Test + public void createAndReadAndDeleteLogFilesForCommand() { + final String vmName = "TestVM"; + final String destIp = "DestinationHost"; + final VirtualMachineTO vmTO = Mockito.mock(VirtualMachineTO.class); + final MigrateCommand command = new MigrateCommand(vmName, destIp, false, vmTO, false); + long requestSequence = 10000L; + command.setRequestSequence(requestSequence); + + String logFile = ReconcileCommandUtils.getLogFileNameForCommand(COMMANDS_LOG_PATH, command); + + ReconcileCommandUtils.createLogFileForCommand(COMMANDS_LOG_PATH, command); + Assert.assertTrue((new File(logFile).exists())); + + CommandInfo commandInfo = ReconcileCommandUtils.readLogFileForCommand(logFile); + Assert.assertNotNull(commandInfo); + Assert.assertEquals(command.getClass().getName(), commandInfo.getCommandName()); + Assert.assertEquals(Command.State.CREATED, commandInfo.getState()); + + Command parseCommand = ReconcileCommandUtils.parseCommandInfo(commandInfo); + System.out.println("command state is " + commandInfo); + Assert.assertNotNull(parseCommand); + Assert.assertTrue(parseCommand instanceof MigrateCommand); + Assert.assertEquals(vmName,((MigrateCommand) parseCommand).getVmName()); + Assert.assertEquals(destIp,((MigrateCommand) parseCommand).getDestinationIp()); + + ReconcileCommandUtils.updateLogFileForCommand(COMMANDS_LOG_PATH, command, Command.State.PROCESSING); + CommandInfo newCommandInfo = ReconcileCommandUtils.readLogFileForCommand(logFile); + System.out.println("new command state is " + newCommandInfo); + Assert.assertNotNull(newCommandInfo); + Assert.assertEquals(command.getClass().getName(), newCommandInfo.getCommandName()); + Assert.assertEquals(Command.State.PROCESSING, newCommandInfo.getState()); + + ReconcileCommandUtils.deleteLogFileForCommand(COMMANDS_LOG_PATH, command); + Assert.assertFalse((new File(logFile).exists())); + } +} diff --git a/debian/cloudstack-management.postinst b/debian/cloudstack-management.postinst index fadb7e57eca..d5d50a4718c 100755 --- a/debian/cloudstack-management.postinst +++ b/debian/cloudstack-management.postinst @@ -56,6 +56,8 @@ if [ "$1" = configure ]; then chmod 0640 ${CONFDIR}/${DBPROPS} chgrp cloud ${CONFDIR}/${DBPROPS} chown -R cloud:cloud /var/log/cloudstack/management + chown -R cloud:cloud /usr/share/cloudstack-management/templates + find /usr/share/cloudstack-management/templates -type d -exec chmod 0770 {} \; ln -sf ${CONFDIR}/log4j-cloud.xml ${CONFDIR}/log4j2.xml diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 94c73d8f4d6..238a78e89af 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -309,4 +309,8 @@ public interface VirtualMachineManager extends Manager { Map getDiskOfferingSuitabilityForVm(long vmId, List diskOfferingIds); + void checkDeploymentPlan(VirtualMachine virtualMachine, VirtualMachineTemplate template, + ServiceOffering serviceOffering, Account systemAccount, DeploymentPlan plan) + throws InsufficientServerCapacityException; + } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 7950dda4d68..afc33eb5190 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.Set; import com.cloud.exception.ResourceAllocationException; +import com.cloud.utils.Pair; import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; @@ -83,6 +84,17 @@ public interface VolumeOrchestrationService { "The maximum size for a volume (in GiB).", true); + ConfigKey VolumeAllocationAlgorithm = new ConfigKey<>( + String.class, + "volume.allocation.algorithm", + "Advanced", + "random", + "Order in which storage pool within a cluster will be considered for volume allocation. The value can be 'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', or 'firstfitleastconsumed'.", + true, + ConfigKey.Scope.Global, null, null, null, null, null, + ConfigKey.Kind.Select, + "random,firstfit,userdispersing,userconcentratedpod_random,userconcentratedpod_firstfit,firstfitleastconsumed"); + VolumeInfo moveVolume(VolumeInfo volume, long destPoolDcId, Long destPoolPodId, Long destPoolClusterId, HypervisorType dataDiskHyperType) throws ConcurrentOperationException, StorageUnavailableException; @@ -179,4 +191,9 @@ public interface VolumeOrchestrationService { * Unmanage VM volumes */ void unmanageVolumes(long vmId); + + /** + * Retrieves the volume's checkpoints paths to be used in the KVM processor. If there are no checkpoints, it will return an empty list. + */ + Pair, Set> getVolumeCheckpointPathsAndImageStoreUrls(long volumeId, HypervisorType hypervisorType); } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreDriver.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreDriver.java index b197afad863..af3b38b368c 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreDriver.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreDriver.java @@ -45,4 +45,8 @@ public interface DataStoreDriver { boolean canCopy(DataObject srcData, DataObject destData); void resize(DataObject data, AsyncCompletionCallback callback); + + default boolean canDisplayDetails() { + return true; + } } diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java index 4f2a69bc771..7cf2a593db4 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/EndPointSelector.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import com.cloud.hypervisor.Hypervisor; + import java.util.List; public interface EndPointSelector { @@ -39,6 +41,8 @@ public interface EndPointSelector { EndPoint select(DataObject object, StorageAction action, boolean encryptionSupportRequired); + EndPoint selectRandom(long zoneId, Hypervisor.HypervisorType hypervisorType); + List selectAll(DataStore store); List findAllEndpointsForScope(DataStore store); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java index 611d1247c49..ce9f4a67a73 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/ObjectInDataStoreStateMachine.java @@ -32,7 +32,8 @@ public interface ObjectInDataStoreStateMachine extends StateObject