diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000000..3c632f8ba53 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[codespell] +ignore-words = .github/linters/codespell.txt +skip = systemvm/agent/noVNC/*,ui/package.json,ui/package-lock.json,ui/public/js/less.min.js,ui/public/locales/*.json,server/src/test/java/org/apache/cloudstack/network/ssl/CertServiceTest.java,test/integration/smoke/test_ssl_offloading.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0602871ef1..689275cff11 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /plugins/storage/volume/linstor @rp- /plugins/storage/volume/storpool @slavkap +/plugins/storage/volume/ontap @rajiv1 @sandeeplocharla @piyush5 @suryag .pre-commit-config.yaml @jbampton /.github/linters/ @jbampton diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 41b307863fc..cef74aafb4b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -22,8 +22,19 @@ version: 2 updates: - - package-ecosystem: "maven" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "github-actions" + directory: "/" + open-pull-requests-limit: 2 + schedule: + interval: "weekly" + groups: + github-actions-dependencies: + patterns: + - "*" + cooldown: + default-days: 7 + - package-ecosystem: "maven" + directory: "/" schedule: interval: "daily" cooldown: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 84020f4a6b0..4c33a131343 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,7 +30,7 @@ jobs: build: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up JDK 17 uses: actions/setup-java@v5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6957d3f5446..df60179ceb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,7 +217,7 @@ jobs: smoke/test_list_volumes"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index fbd944a758f..88b10ac9178 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -32,7 +32,7 @@ jobs: name: codecov runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 74e59aa821d..31a8746b85a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,7 +35,7 @@ jobs: language: ["actions"] steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: diff --git a/.github/workflows/docker-cloudstack-simulator.yml b/.github/workflows/docker-cloudstack-simulator.yml index af6cbf49f5e..8d23ac449dd 100644 --- a/.github/workflows/docker-cloudstack-simulator.yml +++ b/.github/workflows/docker-cloudstack-simulator.yml @@ -47,7 +47,7 @@ jobs: - name: Set Docker repository name run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set ACS version run: echo "ACS_VERSION=$(grep '' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 224ea2cde80..7ccd6600ab9 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -32,7 +32,7 @@ jobs: name: Main Sonar JaCoCo Build runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 11fe5c06881..895a597659d 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-22.04 steps: - name: Check Out - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install run: | python -m pip install --upgrade pip diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index d71f4b0852d..21b8e197d82 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -30,7 +30,7 @@ jobs: build: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up JDK 17 uses: actions/setup-java@v5 with: diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 31fb671cc58..9f5c3a194bc 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -33,7 +33,7 @@ jobs: name: Sonar JaCoCo Coverage runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: ref: "refs/pull/${{ github.event.number }}/merge" fetch-depth: 0 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 56b04a6f9c9..4580b6bbd5d 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Node uses: actions/setup-node@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cf6f8d39027..755ae125edf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -162,17 +162,15 @@ repos: - id: forbid-submodules - id: mixed-line-ending - id: trailing-whitespace - files: ^(LICENSE|NOTICE)$|\.(bat|cfg|cs|css|gitignore|header|in|install|java|md|properties|py|rb|rc|sh|sql|te|template|txt|ucls|vue|xml|xsl|yaml|yml)$|^cloud-cli/bindir/cloud-tool$|^debian/changelog$ + files: ^(LICENSE|NOTICE)$|README$|\.(bat|cfg|config|cs|css|erb|gitignore|header|in|install|java|md|properties|py|rb|rc|sh|sql|svg|te|template|txt|ucls|vue|xml|xsl|yaml|yml)$|^cloud-cli/bindir/cloud-tool$|^debian/changelog$ args: [--markdown-linebreak-ext=md] exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$ - repo: https://github.com/codespell-project/codespell - rev: v2.4.1 + rev: v2.4.2 hooks: - id: codespell name: run codespell description: Check spelling with codespell - args: [--ignore-words=.github/linters/codespell.txt] - exclude: ^systemvm/agent/noVNC/|^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$|^server/src/test/java/org/apache/cloudstack/network/ssl/CertServiceTest.java$|^test/integration/smoke/test_ssl_offloading.py$ - repo: https://github.com/pycqa/flake8 rev: 7.0.0 hooks: @@ -186,7 +184,7 @@ repos: description: check Markdown files with markdownlint args: [--config=.github/linters/.markdown-lint.yml] types: [markdown] - files: \.(md|mdown|markdown)$ + files: \.md$ - repo: https://github.com/adrienverge/yamllint rev: v1.37.1 hooks: diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 0dc5b8211e0..777bd4bf5cd 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -457,3 +457,6 @@ iscsi.session.cleanup.enabled=false # Instance conversion VIRT_V2V_TMPDIR env var #convert.instance.env.virtv2v.tmpdir= + +# Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. +# incremental.snapshot.retry.rebase.wait=60 diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 3364f9708cf..b0926b23a1e 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -885,6 +885,11 @@ public class AgentProperties{ */ public static final Property CREATE_FULL_CLONE = new Property<>("create.full.clone", false); + /** + * Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. + * */ + public static final Property INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60); + public static class Property { private String name; diff --git a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java index f7e4bfea80f..fd8237998a7 100644 --- a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java @@ -26,10 +26,13 @@ public final class BucketTO { private String secretKey; + private long accountId; + public BucketTO(Bucket bucket) { this.name = bucket.getName(); this.accessKey = bucket.getAccessKey(); this.secretKey = bucket.getSecretKey(); + this.accountId = bucket.getAccountId(); } public BucketTO(String name) { @@ -47,4 +50,8 @@ public final class BucketTO { public String getSecretKey() { return this.secretKey; } + + public long getAccountId() { + return this.accountId; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/NicTO.java b/api/src/main/java/com/cloud/agent/api/to/NicTO.java index ca95fcfd679..2ed7d9f9a20 100644 --- a/api/src/main/java/com/cloud/agent/api/to/NicTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/NicTO.java @@ -33,6 +33,7 @@ public class NicTO extends NetworkTO { boolean dpdkEnabled; Integer mtu; Long networkId; + boolean enabled; String networkSegmentName; @@ -154,4 +155,12 @@ public class NicTO extends NetworkTO { public void setNetworkSegmentName(String networkSegmentName) { this.networkSegmentName = networkSegmentName; } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } } diff --git a/api/src/main/java/com/cloud/projects/ProjectService.java b/api/src/main/java/com/cloud/projects/ProjectService.java index 5080cb5a781..d11e9ae0446 100644 --- a/api/src/main/java/com/cloud/projects/ProjectService.java +++ b/api/src/main/java/com/cloud/projects/ProjectService.java @@ -82,7 +82,7 @@ public interface ProjectService { Project updateProject(long id, String name, String displayText, String newOwnerName, Long userId, Role newRole) throws ResourceAllocationException; - boolean addAccountToProject(long projectId, String accountName, String email, Long projectRoleId, Role projectRoleType); + boolean addAccountToProject(long projectId, String accountName, String email, Long projectRoleId, Role projectRoleType) throws ResourceAllocationException; boolean deleteAccountFromProject(long projectId, String accountName); @@ -100,6 +100,6 @@ public interface ProjectService { Project findByProjectAccountIdIncludingRemoved(long projectAccountId); - boolean addUserToProject(Long projectId, String username, String email, Long projectRoleId, Role projectRole); + boolean addUserToProject(Long projectId, String username, String email, Long projectRoleId, Role projectRole) throws ResourceAllocationException; } diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 51737546ffa..bcca229c06b 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -71,7 +71,6 @@ import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; -import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.alert.Alert; import com.cloud.capacity.Capacity; @@ -108,14 +107,6 @@ import com.cloud.vm.VirtualMachineProfile; public interface ManagementService { static final String Name = "management-server"; - ConfigKey JsInterpretationEnabled = new ConfigKey<>("Hidden" - , Boolean.class - , "js.interpretation.enabled" - , "false" - , "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules." - , false - , ConfigKey.Scope.Global); - /** * returns the a map of the names/values in the configuration table * @@ -534,6 +525,4 @@ public interface ManagementService { boolean removeManagementServer(RemoveManagementServerCmd cmd); - void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue); - } diff --git a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java index db702a61f2b..7d5b2d7c57d 100644 --- a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java +++ b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java @@ -23,9 +23,10 @@ import org.apache.cloudstack.api.InternalIdentity; public interface VMTemplateStorageResourceAssoc extends InternalIdentity { public static enum Status { - UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED + UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, LIMIT_REACHED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED } + List ERROR_DOWNLOAD_STATES = List.of(Status.DOWNLOAD_ERROR, Status.ABANDONED, Status.LIMIT_REACHED, Status.UNKNOWN); List PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS); String getInstallPath(); diff --git a/api/src/main/java/com/cloud/user/AccountService.java b/api/src/main/java/com/cloud/user/AccountService.java index 30919e5b782..4145e2b89eb 100644 --- a/api/src/main/java/com/cloud/user/AccountService.java +++ b/api/src/main/java/com/cloud/user/AccountService.java @@ -138,6 +138,8 @@ public interface AccountService { Long finalizeAccountId(String accountName, Long domainId, Long projectId, boolean enabledOnly); + Long finalizeAccountId(Long accountId, String accountName, Long domainId, Long projectId); + /** * returns the user account object for a given user id * @param userId user id diff --git a/api/src/main/java/com/cloud/user/ResourceLimitService.java b/api/src/main/java/com/cloud/user/ResourceLimitService.java index 738e593582b..9c493fb383c 100644 --- a/api/src/main/java/com/cloud/user/ResourceLimitService.java +++ b/api/src/main/java/com/cloud/user/ResourceLimitService.java @@ -30,6 +30,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; import com.cloud.template.VirtualMachineTemplate; +import org.apache.cloudstack.resourcelimit.Reserver; public interface ResourceLimitService { @@ -191,6 +192,7 @@ public interface ResourceLimitService { */ public void checkResourceLimit(Account account, ResourceCount.ResourceType type, long... count) throws ResourceAllocationException; public void checkResourceLimitWithTag(Account account, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; + public void checkResourceLimitWithTag(Account account, Long domainId, boolean considerSystemAccount, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; /** * Gets the count of resources for a resource type and account @@ -251,12 +253,12 @@ public interface ResourceLimitService { List getResourceLimitStorageTags(DiskOffering diskOffering); void updateTaggedResourceLimitsAndCountsForAccounts(List responses, String tag); void updateTaggedResourceLimitsAndCountsForDomains(List responses, String tag); - void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; - + void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List reservations) throws ResourceAllocationException; + List getResourceLimitStorageTagsForResourceCountOperation(Boolean display, DiskOffering diskOffering); void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean display, Long currentSize, Long newSize, - DiskOffering currentOffering, DiskOffering newOffering) throws ResourceAllocationException; + DiskOffering currentOffering, DiskOffering newOffering, List reservations) throws ResourceAllocationException; - void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; + void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List reservations) throws ResourceAllocationException; void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); @@ -273,25 +275,23 @@ public interface ResourceLimitService { void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); - void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, List reservations) throws ResourceAllocationException; void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); void checkVmResourceLimitsForServiceOfferingChange(Account owner, Boolean display, Long currentCpu, Long newCpu, - Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template, List reservations) throws ResourceAllocationException; void checkVmResourceLimitsForTemplateChange(Account owner, Boolean display, ServiceOffering offering, - VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate) throws ResourceAllocationException; + VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate, List reservations) throws ResourceAllocationException; - void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException; void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); - void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException; void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); - void checkVmGpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu) throws ResourceAllocationException; void incrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu); void decrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu); + long recalculateDomainResourceCount(final long domainId, final ResourceType type, String tag); } diff --git a/api/src/main/java/com/cloud/vm/Nic.java b/api/src/main/java/com/cloud/vm/Nic.java index afc44b8d39f..cc0b294205c 100644 --- a/api/src/main/java/com/cloud/vm/Nic.java +++ b/api/src/main/java/com/cloud/vm/Nic.java @@ -162,4 +162,6 @@ public interface Nic extends Identity, InternalIdentity { String getIPv6Address(); Integer getMtu(); + + boolean isEnabled(); } diff --git a/api/src/main/java/com/cloud/vm/NicProfile.java b/api/src/main/java/com/cloud/vm/NicProfile.java index a0c80ceb1bf..d3ed7b4b87d 100644 --- a/api/src/main/java/com/cloud/vm/NicProfile.java +++ b/api/src/main/java/com/cloud/vm/NicProfile.java @@ -52,6 +52,7 @@ public class NicProfile implements InternalIdentity, Serializable { boolean defaultNic; Integer networkRate; boolean isSecurityGroupEnabled; + boolean enabled; Integer orderIndex; @@ -87,6 +88,7 @@ public class NicProfile implements InternalIdentity, Serializable { broadcastType = network.getBroadcastDomainType(); trafficType = network.getTrafficType(); format = nic.getAddressFormat(); + enabled = nic.isEnabled(); iPv4Address = nic.getIPv4Address(); iPv4Netmask = nic.getIPv4Netmask(); @@ -414,6 +416,14 @@ public class NicProfile implements InternalIdentity, Serializable { this.ipv4AllocationRaceCheck = ipv4AllocationRaceCheck; } + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + // // OTHER METHODS // diff --git a/api/src/main/java/com/cloud/vm/UserVmService.java b/api/src/main/java/com/cloud/vm/UserVmService.java index 01f11b73cd4..67aa0534a5f 100644 --- a/api/src/main/java/com/cloud/vm/UserVmService.java +++ b/api/src/main/java/com/cloud/vm/UserVmService.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.api.command.user.vm.ScaleVMCmd; import org.apache.cloudstack.api.command.user.vm.StartVMCmd; import org.apache.cloudstack.api.command.user.vm.UpdateDefaultNicForVMCmd; import org.apache.cloudstack.api.command.user.vm.UpdateVMCmd; +import org.apache.cloudstack.api.command.user.vm.UpdateVmNicCmd; import org.apache.cloudstack.api.command.user.vm.UpdateVmNicIpCmd; import org.apache.cloudstack.api.command.user.vm.UpgradeVMCmd; import org.apache.cloudstack.api.command.user.vmgroup.CreateVMGroupCmd; @@ -152,6 +153,8 @@ public interface UserVmService { */ UserVm updateNicIpForVirtualMachine(UpdateVmNicIpCmd cmd); + UserVm updateVirtualMachineNic(UpdateVmNicCmd cmd); + UserVm recoverVirtualMachine(RecoverVMCmd cmd) throws ResourceAllocationException; /** @@ -524,6 +527,7 @@ public interface UserVmService { * @param userId user ID * @param serviceOffering service offering for the imported VM * @param sshPublicKey ssh key for the imported VM + * @param guestOsId guest OS ID for the imported VM (if not passed, then the guest OS of the template will be used) * @param hostName the name for the imported VM * @param hypervisorType hypervisor type for the imported VM * @param customParameters details for the imported VM @@ -533,7 +537,7 @@ public interface UserVmService { * @throws InsufficientCapacityException in case of errors */ UserVm importVM(final DataCenter zone, final Host host, final VirtualMachineTemplate template, final String instanceNameInternal, final String displayName, final Account owner, final String userData, final Account caller, final Boolean isDisplayVm, final String keyboard, - final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKey, + final long accountId, final long userId, final ServiceOffering serviceOffering, final String sshPublicKey, final Long guestOsId, final String hostName, final HypervisorType hypervisorType, final Map customParameters, final VirtualMachine.PowerState powerState, final LinkedHashMap> networkNicMap) throws InsufficientCapacityException; diff --git a/api/src/main/java/com/cloud/vm/VirtualMachine.java b/api/src/main/java/com/cloud/vm/VirtualMachine.java index d244de7115e..41c9a864c9d 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachine.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachine.java @@ -124,6 +124,9 @@ public interface VirtualMachine extends RunningOn, ControlledEntity, Partition, s_fsm.addTransition(new Transition(State.Stopping, VirtualMachine.Event.StopRequested, State.Stopping, null)); s_fsm.addTransition(new Transition(State.Stopping, VirtualMachine.Event.AgentReportShutdowned, State.Stopped, Arrays.asList(new Impact[]{Impact.USAGE}))); s_fsm.addTransition(new Transition(State.Expunging, VirtualMachine.Event.OperationFailed, State.Expunging,null)); + // Note: In addition to the Stopped -> Error transition for failed VM creation, + // a VM can also transition from Expunging to Error on OperationFailedToError. + s_fsm.addTransition(new Transition(State.Expunging, VirtualMachine.Event.OperationFailedToError, State.Error, null)); s_fsm.addTransition(new Transition(State.Expunging, VirtualMachine.Event.ExpungeOperation, State.Expunging,null)); s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.DestroyRequested, State.Expunging, null)); s_fsm.addTransition(new Transition(State.Error, VirtualMachine.Event.ExpungeOperation, State.Expunging, null)); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 4d33ba859a5..e2ebb242cbf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -127,8 +127,8 @@ public enum ApiCommandResourceType { } public static ApiCommandResourceType fromString(String value) { - if (StringUtils.isNotEmpty(value) && EnumUtils.isValidEnum(ApiCommandResourceType.class, value)) { - return valueOf(value); + if (StringUtils.isNotBlank(value) && EnumUtils.isValidEnumIgnoreCase(ApiCommandResourceType.class, value)) { + return EnumUtils.getEnumIgnoreCase(ApiCommandResourceType.class, value); } return null; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 3d827641358..9ec85734233 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -20,6 +20,7 @@ public class ApiConstants { public static final String ACCOUNT = "account"; public static final String ACCOUNTS = "accounts"; public static final String ACCOUNT_NAME = "accountname"; + public static final String ACCOUNT_STATE_TO_SHOW = "accountstatetoshow"; public static final String ACCOUNT_TYPE = "accounttype"; public static final String ACCOUNT_ID = "accountid"; public static final String ACCOUNT_IDS = "accountids"; @@ -508,6 +509,7 @@ public class ApiConstants { public static final String REPAIR = "repair"; public static final String REPETITION_ALLOWED = "repetitionallowed"; public static final String REQUIRES_HVM = "requireshvm"; + public static final String RESERVED_RESOURCE_DETAILS = "reservedresourcedetails"; public static final String RESOURCES = "resources"; public static final String RESOURCE_COUNT = "resourcecount"; public static final String RESOURCE_NAME = "resourcename"; diff --git a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java index 338c1e738df..b0738cf78e1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java +++ b/api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java @@ -343,6 +343,8 @@ public interface ResponseGenerator { UserVm findUserVmById(Long vmId); + UserVm findUserVmByNicId(Long nicId); + Volume findVolumeById(Long volumeId); Account findAccountByNameDomain(String accountName, Long domainId); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index f7940460d6c..50ccfbd69c5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.GuestOSResponse; import org.apache.cloudstack.api.response.HostResponse; import org.apache.cloudstack.api.response.NetworkResponse; import org.apache.cloudstack.api.response.StoragePoolResponse; @@ -171,6 +172,13 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "(only for importing VMs from VMware to KVM) optional - if true, forces virt-v2v conversions to write directly on the provided storage pool (avoid using temporary conversion pool).") private Boolean forceConvertToPool; + @Parameter(name = ApiConstants.OS_ID, + type = CommandType.UUID, + entityType = GuestOSResponse.class, + since = "4.22.1", + description = "(only for importing VMs from VMware to KVM) optional - the ID of the guest OS for the imported VM.") + private Long guestOsId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -268,6 +276,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return BooleanUtils.toBooleanDefaultIfNull(forceConvertToPool, false); } + public Long getGuestOsId() { + return guestOsId; + } + @Override public String getEventDescription() { String vmName = getName(); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java index 85e7b0af319..63a0a6ca51e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.user.account; import java.util.List; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.BaseCmd; @@ -106,7 +107,7 @@ public class AddAccountToProjectCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// @Override - public void execute() { + public void execute() throws ResourceAllocationException { if (accountName == null && email == null) { throw new InvalidParameterValueException("Either accountName or email is required"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java index 69832d4dfdc..683522039b1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.account; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; @@ -111,7 +112,7 @@ public class AddUserToProjectCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// @Override - public void execute() { + public void execute() throws ResourceAllocationException { validateInput(); boolean result = _projectService.addUserToProject(getProjectId(), getUsername(), getEmail(), getProjectRoleId(), getRoleType()); if (result) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java index 703a1b2e880..4644687817d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.api.command.user.backup; import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; @@ -53,6 +54,7 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, @@ -60,12 +62,14 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { description = "ID of the Instance backup") private Long backupId; + @ACL @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.STRING, required = true, description = "ID of the volume backed up") private String volumeUuid; + @ACL @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java index 4ff2ebf0a66..d558e4c4f24 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/guest/ListGuestOsCmd.java @@ -45,6 +45,11 @@ public class ListGuestOsCmd extends BaseListCmd { @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = GuestOSResponse.class, description = "List by OS type ID") private Long id; + @Parameter(name = ApiConstants.IDS, type = CommandType.LIST, collectionType = CommandType.UUID, + entityType = GuestOSResponse.class, since = "4.22.1", + description = "Comma separated list of OS types") + private List ids; + @Parameter(name = ApiConstants.OS_CATEGORY_ID, type = CommandType.UUID, entityType = GuestOSCategoryResponse.class, description = "List by OS Category ID") private Long osCategoryId; @@ -63,6 +68,10 @@ public class ListGuestOsCmd extends BaseListCmd { return id; } + public List getIds() { + return ids; + } + public Long getOsCategoryId() { return osCategoryId; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index b55d1b234f1..2c840183113 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.job; import java.util.Date; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; @@ -40,6 +41,12 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "The id of the management server", since="4.19") private Long managementServerId; + @Parameter(name = ApiConstants.RESOURCE_ID, validations = {ApiArgValidator.UuidString}, type = CommandType.STRING, description = "the ID of the resource associated with the job", since="4.22.1") + private String resourceId; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") + private String resourceType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -52,6 +59,14 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { return managementServerId; } + public String getResourceId() { + return resourceId; + } + + public String getResourceType() { + return resourceType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java index 93a44375721..5c3b0084574 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java @@ -16,8 +16,8 @@ // under the License. package org.apache.cloudstack.api.command.user.job; - import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; @@ -34,9 +34,15 @@ public class QueryAsyncJobResultCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, required = true, description = "The ID of the asynchronous job") + @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, description = "The ID of the asynchronous job") private Long id; + @Parameter(name = ApiConstants.RESOURCE_ID, validations = {ApiArgValidator.UuidString}, type = CommandType.STRING, description = "the ID of the resource associated with the job", since="4.22.1") + private String resourceId; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") + private String resourceType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -45,6 +51,14 @@ public class QueryAsyncJobResultCmd extends BaseCmd { return id; } + public String getResourceId() { + return resourceId; + } + + public String getResourceType() { + return resourceType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index e17ba9c2d70..a719062e1bc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -51,6 +51,7 @@ public class CreateVMFromBackupCmd extends BaseDeployVMCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVmNicCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVmNicCmd.java new file mode 100644 index 00000000000..363273a4670 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVmNicCmd.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.user.vm; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ACL; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.NicResponse; +import org.apache.cloudstack.api.response.UserVmResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.user.Account; +import com.cloud.uservm.UserVm; + +import java.util.ArrayList; +import java.util.EnumSet; + +@APICommand(name = "updateVmNic", description = "Updates the specified VM NIC", responseObject = NicResponse.class, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, + authorized = { RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User }) +public class UpdateVmNicCmd extends BaseAsyncCmd { + + @ACL + @Parameter(name = ApiConstants.NIC_ID, type = CommandType.UUID, entityType = NicResponse.class, required = true, description = "NIC ID") + private Long nicId; + + @Parameter(name = ApiConstants.ENABLED, type = CommandType.BOOLEAN, description = "If true, sets the NIC state to UP; otherwise, sets the NIC state to DOWN") + private Boolean enabled; + + public Long getNicId() { + return nicId; + } + + public Boolean isEnabled() { + return enabled; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_NIC_UPDATE; + } + + @Override + public String getEventDescription() { + return String.format("Updating NIC %s.", getResourceUuid(ApiConstants.NIC_ID)); + } + + @Override + public long getEntityOwnerId() { + UserVm vm = _responseGenerator.findUserVmByNicId(nicId); + if (vm == null) { + return Account.ACCOUNT_ID_SYSTEM; + } + return vm.getAccountId(); + } + + @Override + public void execute() { + CallContext.current().setEventDetails(String.format("NIC ID: %s", getResourceUuid(ApiConstants.NIC_ID))); + + UserVm result = _userVmService.updateVirtualMachineNic(this); + + ArrayList dc = new ArrayList<>(); + dc.add(ApiConstants.VMDetails.valueOf("nics")); + EnumSet details = EnumSet.copyOf(dc); + + if (result != null){ + UserVmResponse response = _responseGenerator.createUserVmResponse(ResponseObject.ResponseView.Restricted, "virtualmachine", details, result).get(0); + response.setResponseName(getCommandName()); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to update NIC from VM."); + } + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 5bcf3a14117..78de0564848 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -109,6 +110,13 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC description = "The ID of the Instance; to be used with snapshot Id, Instance to which the volume gets attached after creation") private Long virtualMachineId; + @Parameter(name = ApiConstants.STORAGE_ID, + type = CommandType.UUID, + entityType = StoragePoolResponse.class, + description = "Storage pool ID to create the volume in. Cannot be used with the snapshotid parameter.", + authorized = {RoleType.Admin}) + private Long storageId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -153,6 +161,13 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC return projectId; } + public Long getStorageId() { + if (snapshotId != null && storageId != null) { + throw new IllegalArgumentException("StorageId parameter cannot be specified with the SnapshotId parameter."); + } + return storageId; + } + public Boolean getDisplayVolume() { return displayVolume; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java index 11930bcbab6..9b5d25aa0dd 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateStaticRouteCmd.java @@ -46,7 +46,6 @@ public class CreateStaticRouteCmd extends BaseAsyncCreateCmd { @Parameter(name = ApiConstants.GATEWAY_ID, type = CommandType.UUID, entityType = PrivateGatewayResponse.class, - required = true, description = "The gateway ID we are creating static route for. Mutually exclusive with the nexthop parameter") private Long gatewayId; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java index fdf1e87df50..911839da405 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java @@ -85,6 +85,12 @@ public class ExtensionResponse extends BaseResponse { @Param(description = "Removal timestamp of the extension, if applicable") private Date removed; + @SerializedName(ApiConstants.RESERVED_RESOURCE_DETAILS) + @Param(description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + public ExtensionResponse(String id, String name, String description, String type) { this.id = id; this.name = name; @@ -179,4 +185,8 @@ public class ExtensionResponse extends BaseResponse { public void setRemoved(Date removed) { this.removed = removed; } + + public void setReservedResourceDetails(String reservedResourceDetails) { + this.reservedResourceDetails = reservedResourceDetails; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java index f992514b8db..92f25e370fb 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NicResponse.java @@ -146,6 +146,10 @@ public class NicResponse extends BaseResponse { @Param(description = "Public IP address associated with this NIC via Static NAT rule") private String publicIp; + @SerializedName(ApiConstants.ENABLED) + @Param(description = "whether the NIC is enabled or not") + private Boolean isEnabled; + public void setVmId(String vmId) { this.vmId = vmId; } @@ -416,4 +420,12 @@ public class NicResponse extends BaseResponse { public void setPublicIp(String publicIp) { this.publicIp = publicIp; } + + public Boolean getEnabled() { + return isEnabled; + } + + public void setEnabled(Boolean enabled) { + isEnabled = enabled; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java index 195323b741d..3f2dc045e87 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/UnmanagedInstanceResponse.java @@ -51,6 +51,14 @@ public class UnmanagedInstanceResponse extends BaseResponse { @Param(description = "The name of the host to which Instance belongs") private String hostName; + @SerializedName(ApiConstants.HYPERVISOR) + @Param(description = "The hypervisor to which Instance belongs") + private String hypervisor; + + @SerializedName(ApiConstants.HYPERVISOR_VERSION) + @Param(description = "The hypervisor version of the host to which Instance belongs") + private String hypervisorVersion; + @SerializedName(ApiConstants.POWER_STATE) @Param(description = "The power state of the Instance") private String powerState; @@ -140,6 +148,22 @@ public class UnmanagedInstanceResponse extends BaseResponse { this.hostName = hostName; } + public String getHypervisor() { + return hypervisor; + } + + public void setHypervisor(String hypervisor) { + this.hypervisor = hypervisor; + } + + public String getHypervisorVersion() { + return hypervisorVersion; + } + + public void setHypervisorVersion(String hypervisorVersion) { + this.hypervisorVersion = hypervisorVersion; + } + public String getPowerState() { return powerState; } diff --git a/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java b/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java index f50f841ed74..a01131278a7 100644 --- a/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java +++ b/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java @@ -17,8 +17,11 @@ package org.apache.cloudstack.extension; +import java.util.List; + public interface ExtensionHelper { Long getExtensionIdForCluster(long clusterId); Extension getExtension(long id); Extension getExtensionForCluster(long clusterId); + List getExtensionReservedResourceDetails(long extensionId); } diff --git a/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java b/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java new file mode 100644 index 00000000000..6b3f57b6aa5 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.resourcelimit; + +/** + * Interface implemented by CheckedReservation. + *

+ * This is defined in cloud-api to allow methods declared in modules that do not depend on cloud-server + * to receive CheckedReservations as parameters. + */ +public interface Reserver extends AutoCloseable { + + void close(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java index 5f69f3e46e7..13f01b4944a 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -25,17 +25,28 @@ import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.VolumeForImportResponse; import org.apache.cloudstack.api.response.VolumeResponse; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; import java.util.Arrays; import java.util.List; -public interface VolumeImportUnmanageService extends PluggableService { +public interface VolumeImportUnmanageService extends PluggableService, Configurable { List SUPPORTED_HYPERVISORS = Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware); List SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD); + Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint); + + ConfigKey AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class, + "allow.import.volume.with.backing.file", + "Advanced", + "false", + "If enabled, allows QCOW2 volumes with backing files to be imported or unmanaged", + true, + ConfigKey.Scope.Global, + null); ListResponse listVolumesForImport(ListVolumesForImportCmd cmd); diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java index bba97dff71c..cbb7c4de698 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedInstanceTO.java @@ -55,6 +55,9 @@ public class UnmanagedInstanceTO { private String hostName; + private String hypervisorType; + private String hostHypervisorVersion; + private List disks; private List nics; @@ -168,6 +171,22 @@ public class UnmanagedInstanceTO { this.hostName = hostName; } + public String getHypervisorType() { + return hypervisorType; + } + + public void setHypervisorType(String hypervisorType) { + this.hypervisorType = hypervisorType; + } + + public String getHostHypervisorVersion() { + return hostHypervisorVersion; + } + + public void setHostHypervisorVersion(String hostHypervisorVersion) { + this.hostHypervisorVersion = hostHypervisorVersion; + } + public List getDisks() { return disks; } diff --git a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java index b6233b9c270..e1963313eb6 100644 --- a/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java +++ b/api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java @@ -26,6 +26,8 @@ import static com.cloud.hypervisor.Hypervisor.HypervisorType.VMware; public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService, PluggableService, Configurable { + String VM_IMPORT_DEFAULT_TEMPLATE_NAME = "system-default-vm-import-dummy-template.iso"; + String KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME = "kvm-default-vm-import-dummy-template"; ConfigKey UnmanageVMPreserveNic = new ConfigKey<>("Advanced", Boolean.class, "unmanage.vm.preserve.nics", "false", "If set to true, do not remove VM nics (and its MAC addresses) when unmanaging a VM, leaving them allocated but not reserved. " + "If set to false, nics are removed and MAC addresses can be reassigned", true, ConfigKey.Scope.Zone); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java index f100822b8c7..cd0390aa268 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.test; +import com.cloud.exception.ResourceAllocationException; import junit.framework.Assert; import junit.framework.TestCase; @@ -149,6 +150,8 @@ public class AddAccountToProjectCmdTest extends TestCase { addAccountToProjectCmd.execute(); } catch (InvalidParameterValueException exception) { Assert.assertEquals("Either accountName or email is required", exception.getLocalizedMessage()); + } catch (ResourceAllocationException exception) { + Assert.fail(); } } diff --git a/client/pom.xml b/client/pom.xml index b8dffe65d4f..7118f455ab5 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -121,6 +121,11 @@ cloud-plugin-storage-volume-adaptive ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-volume-ontap + ${project.version} + org.apache.cloudstack cloud-plugin-storage-volume-solidfire diff --git a/core/src/main/java/com/cloud/agent/api/UpdateVmNicAnswer.java b/core/src/main/java/com/cloud/agent/api/UpdateVmNicAnswer.java new file mode 100644 index 00000000000..0af6d7fa3a3 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UpdateVmNicAnswer.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +public class UpdateVmNicAnswer extends Answer { + public UpdateVmNicAnswer() { + } + + public UpdateVmNicAnswer(UpdateVmNicCommand cmd, boolean success, String result) { + super(cmd, success, result); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/UpdateVmNicCommand.java b/core/src/main/java/com/cloud/agent/api/UpdateVmNicCommand.java new file mode 100644 index 00000000000..a5c5faf32fe --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/UpdateVmNicCommand.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package com.cloud.agent.api; + +public class UpdateVmNicCommand extends Command { + + String nicMacAddress; + String instanceName; + Boolean enabled; + + @Override + public boolean executeInSequence() { + return true; + } + + protected UpdateVmNicCommand() { + } + + public UpdateVmNicCommand(String nicMacAdderss, String instanceName, Boolean enabled) { + this.nicMacAddress = nicMacAdderss; + this.instanceName = instanceName; + this.enabled = enabled; + } + + public String getNicMacAddress() { + return nicMacAddress; + } + + public String getVmName() { + return instanceName; + } + + public Boolean isEnabled() { + return enabled; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 0c6373134b1..1c5eb7b9a9a 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -140,7 +140,7 @@ public class DownloadAnswer extends Answer { } public Long getTemplateSize() { - return templateSize; + return templateSize == 0 ? templatePhySicalSize : templateSize; } public void setTemplatePhySicalSize(long templatePhySicalSize) { diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 6fe001de72c..71c329796d1 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -52,6 +52,7 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.net.Proxy; /** @@ -125,6 +126,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index 95ed0d1e76d..0dad2564779 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -18,8 +18,11 @@ // package com.cloud.storage.template; + import com.cloud.storage.StorageLayer; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; @@ -59,6 +62,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index 8719947cb4f..6608754073a 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -44,6 +44,7 @@ import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.lang3.StringUtils; import com.cloud.storage.StorageLayer; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); @@ -95,6 +96,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } @@ -141,6 +143,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem continue; } HeadMethod headMethod = new HeadMethod(downloadUrl); + headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { if (client.executeMethod(headMethod) != HttpStatus.SC_OK) { continue; diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f5ad5fbea2c..972c2eaf7bb 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command { private List backupVolumesUUIDs; private List restoreVolumePools; private List restoreVolumePaths; + private List restoreVolumeSizes; private List backupFiles; private String diskType; private Boolean vmExists; @@ -92,6 +93,14 @@ public class RestoreBackupCommand extends Command { this.restoreVolumePaths = restoreVolumePaths; } + public List getRestoreVolumeSizes() { + return restoreVolumeSizes; + } + + public void setRestoreVolumeSizes(List restoreVolumeSizes) { + this.restoreVolumeSizes = restoreVolumeSizes; + } + public List getBackupFiles() { return backupFiles; } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index a1485463eaa..05619e5632b 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -21,6 +21,7 @@ package org.apache.cloudstack.direct.download; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -33,6 +34,7 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -128,15 +130,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown */ protected File createTemporaryDirectoryAndFile(String downloadDir) { createFolder(downloadDir); - return new File(downloadDir + File.separator + getFileNameFromUrl()); + return new File(downloadDir + File.separator + getTemporaryFileName()); } /** - * Return filename from url + * Return filename from the temporary download file */ - public String getFileNameFromUrl() { - String[] urlParts = url.split("/"); - return urlParts[urlParts.length - 1]; + public String getTemporaryFileName() { + return String.format("%s.%s", UUID.randomUUID(), FilenameUtils.getExtension(url)); } @Override diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index c4a802ecdbc..99b84bb645c 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.direct.download; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -32,6 +33,7 @@ import java.util.Map; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; @@ -39,6 +41,7 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.io.IOUtils; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -68,6 +71,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(this.isFollowRedirects()); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); @@ -111,6 +115,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { public boolean checkUrl(String url) { HeadMethod httpHead = new HeadMethod(url); httpHead.setFollowRedirects(this.isFollowRedirects()); + httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { int responseCode = client.executeMethod(httpHead); if (responseCode != HttpStatus.SC_OK) { diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java index 2050b9ef09f..854c310cde9 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -97,7 +97,7 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(), getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath); try { - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getTemporaryFileName()); File f = new File(getDownloadedFilePath()); if (f.exists()) { f.delete(); diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 21184ef07fe..6b0959b78ff 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -69,7 +69,7 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid); Script.runSimpleBashScript(mount); String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId()); - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getTemporaryFileName()); Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath()); Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return new Pair<>(true, getDownloadedFilePath()); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index 253a2607a72..98db75f3902 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -19,6 +19,9 @@ package org.apache.cloudstack.storage.command; +import com.cloud.configuration.Resource; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; + public class TemplateOrVolumePostUploadCommand { long entityId; @@ -185,6 +188,11 @@ public class TemplateOrVolumePostUploadCommand { this.description = description; } + public void setDefaultMaxSecondaryStorageInBytes(long defaultMaxSecondaryStorageInBytes) { + this.defaultMaxSecondaryStorageInGB = defaultMaxSecondaryStorageInBytes != Resource.RESOURCE_UNLIMITED ? + ByteScaleUtils.bytesToGibibytes(defaultMaxSecondaryStorageInBytes) : Resource.RESOURCE_UNLIMITED; + } + public void setDefaultMaxSecondaryStorageInGB(long defaultMaxSecondaryStorageInGB) { this.defaultMaxSecondaryStorageInGB = defaultMaxSecondaryStorageInGB; } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java index 9e6b76e467f..f78744046f7 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java @@ -28,6 +28,7 @@ public class UploadStatusCommand extends Command { } private String entityUuid; private EntityType entityType; + private Boolean abort; protected UploadStatusCommand() { } @@ -37,6 +38,11 @@ public class UploadStatusCommand extends Command { this.entityType = entityType; } + public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) { + this(entityUuid, entityType); + this.abort = abort; + } + public String getEntityUuid() { return entityUuid; } @@ -45,6 +51,10 @@ public class UploadStatusCommand extends Command { return entityType; } + public Boolean getAbort() { + return abort; + } + @Override public boolean executeInSequence() { return false; diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 70240454689..57dc1b7bf72 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -230,6 +230,8 @@ public interface VirtualMachineManager extends Manager { Boolean updateDefaultNicForVM(VirtualMachine vm, Nic nic, Nic defaultNic); + boolean updateVmNic(VirtualMachine vm, Nic nic, Boolean enabled) throws ResourceUnavailableException; + /** * @param vm * @param network diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 947cbd8e618..615320e1788 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -310,7 +310,7 @@ public interface NetworkOrchestrationService { void removeDhcpServiceInSubnet(Nic nic); - boolean resourceCountNeedsUpdate(NetworkOffering ntwkOff, ACLType aclType); + boolean isResourceCountUpdateNeeded(NetworkOffering networkOffering); void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 6f8c4630456..a55219511cb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -120,7 +120,7 @@ public interface VolumeOrchestrationService { void destroyVolume(Volume volume); DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId); + Account owner, Long deviceId, boolean incrementResourceCount); VolumeInfo createVolumeOnPrimaryStorage(VirtualMachine vm, VolumeInfo volume, HypervisorType rootDiskHyperType, StoragePool storagePool) throws NoTransitionException; diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index ddc8153d739..3ae94479cea 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -21,6 +21,7 @@ import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Cluster; import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.component.Manager; import com.cloud.vm.VMInstanceVO; import org.apache.cloudstack.framework.config.ConfigKey; @@ -32,6 +33,8 @@ import java.util.List; */ public interface HighAvailabilityManager extends Manager { + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index e8796fb0252..71ecc73f325 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.vm; +import static com.cloud.configuration.ConfigurationManagerImpl.EXPOSE_ERRORS_TO_USER; import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS; import java.lang.reflect.Field; @@ -49,7 +50,6 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; - import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -153,6 +153,8 @@ import com.cloud.agent.api.UnPlugNicAnswer; import com.cloud.agent.api.UnPlugNicCommand; import com.cloud.agent.api.UnmanageInstanceCommand; import com.cloud.agent.api.UnregisterVMCommand; +import com.cloud.agent.api.UpdateVmNicAnswer; +import com.cloud.agent.api.UpdateVmNicCommand; import com.cloud.agent.api.VmDiskStatsEntry; import com.cloud.agent.api.VmNetworkStatsEntry; import com.cloud.agent.api.VmStatsEntry; @@ -307,7 +309,6 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.gson.Gson; - public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { public static final String VM_WORK_JOB_HANDLER = VirtualMachineManagerImpl.class.getSimpleName(); @@ -585,7 +586,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac Long deviceId = dataDiskDeviceIds.get(index++); String volumeName = deviceId == null ? "DATA-" + persistedVm.getId() : "DATA-" + persistedVm.getId() + "-" + String.valueOf(deviceId); volumeMgr.allocateRawVolume(Type.DATADISK, volumeName, dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId); + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId, true); } } if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { @@ -595,7 +596,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf( diskNumber), diskOffering, diskOfferingSize, null, null, - persistedVm, dataDiskTemplate, owner, diskNumber); + persistedVm, dataDiskTemplate, owner, diskNumber, true); diskNumber++; } } @@ -625,7 +626,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac String rootVolumeName = String.format("ROOT-%s", vm.getId()); if (template.getFormat() == ImageFormat.ISO) { volumeMgr.allocateRawVolume(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), - rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null); + rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null, true); } else if (Arrays.asList(ImageFormat.BAREMETAL, ImageFormat.EXTERNAL).contains(template.getFormat())) { logger.debug("{} has format [{}]. Skipping ROOT volume [{}] allocation.", template, template.getFormat(), rootVolumeName); } else { @@ -931,10 +932,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void start(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { try { advanceStart(vmUuid, params, planToDeploy, planner); - } catch (ConcurrentOperationException | InsufficientCapacityException e) { - throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } catch (ConcurrentOperationException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to concurrent operation.", vmUuid), e).add(VirtualMachine.class, vmUuid); + } catch (final InsufficientCapacityException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to insufficient capacity.", vmUuid), e).add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { - if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){ + if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)) { Account callingAccount = CallContext.current().getCallingAccount(); String errorSuffix = (callingAccount != null && callingAccount.getType() == Account.Type.ADMIN) ? String.format("Failure: %s", e.getMessage()) : @@ -1365,6 +1378,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); + Throwable lastKnownError = null; boolean canRetry = true; ExcludeList avoids = null; try { @@ -1388,7 +1402,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac int retry = StartRetry.value(); while (retry-- != 0) { - logger.debug("Instance start attempt #{}", (StartRetry.value() - retry)); + int attemptNumber = StartRetry.value() - retry; + logger.debug("Instance start attempt #{}", attemptNumber); if (reuseVolume) { final List vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); @@ -1454,8 +1469,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac reuseVolume = false; continue; } - throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(), - areAffinityGroupsAssociated(vmProfile)); + String message = String.format("Unable to create a deployment for %s after %s attempts", vmProfile, attemptNumber); + if (canExposeError(account) && lastKnownError != null) { + message += String.format(" Last known error: %s", lastKnownError.getMessage()); + throw new CloudRuntimeException(message, lastKnownError); + } else { + throw new InsufficientServerCapacityException(message, DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); + } } avoids.addHost(dest.getHost().getId()); @@ -1623,11 +1643,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying"); } } - logger.info("Unable to start VM on {} due to {}", dest.getHost(), (startAnswer == null ? " no start answer" : startAnswer.getDetails())); + String msg = String.format("Unable to start VM on %s due to %s", dest.getHost(), startAnswer == null ? "no start command answer" : startAnswer.getDetails()); + lastKnownError = new ExecutionException(msg); + if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) { + logger.error(msg, lastKnownError); break; } + logger.debug(msg, lastKnownError); } catch (OperationTimedoutException e) { logger.debug("Unable to send the start command to host {} failed to start VM: {}", dest.getHost(), vm); if (e.isActive()) { @@ -1637,6 +1661,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e); } catch (final ResourceUnavailableException e) { logger.warn("Unable to contact resource.", e); + lastKnownError = e; if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; @@ -1693,10 +1718,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } if (startedVm == null) { - throw new CloudRuntimeException("Unable to start Instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details"); + String messageTmpl = "Unable to start Instance '%s' (%s)%s"; + String details; + if (canExposeError(account) && lastKnownError != null) { + details = ": " + lastKnownError.getMessage(); + } else { + details = ", see management server log for details"; + } + String message = String.format(messageTmpl, vm.getHostName(), vm.getUuid(), details); + throw new CloudRuntimeException(message, lastKnownError); } } + private boolean canExposeError(Account account) { + return (account != null && account.getType() == Account.Type.ADMIN) || Boolean.TRUE.equals(EXPOSE_ERRORS_TO_USER.value()); + } + protected void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command) { if (!HypervisorType.External.equals(host.getHypervisorType())) { return; @@ -2180,7 +2217,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) { final VirtualMachine vm = profile.getVirtualMachine(); Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup); updateStopCommandForExternalHypervisorType(vm.getHypervisorType(), profile, stpCmd); if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { @@ -5239,9 +5275,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering) { Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); - details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); - details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); - details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); + + // We need to restore only the customizable parameters. If we save a parameter that is not customizable and attempt + // to restore a VM snapshot, com.cloud.vm.UserVmManagerImpl.validateCustomParameters will fail. + ServiceOffering unfilledOffering = _serviceOfferingDao.findByIdIncludingRemoved(serviceOffering.getId()); + if (unfilledOffering.getCpu() == null) { + details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); + } + if (unfilledOffering.getSpeed() == null) { + details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); + } + if (unfilledOffering.getRamSize() == null) { + details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); + } + List detailList = new ArrayList<>(); for (Map.Entry entry: details.entrySet()) { VMInstanceDetailVO detailVO = new VMInstanceDetailVO(vmId, entry.getKey(), entry.getValue(), true); @@ -6270,6 +6317,80 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac _jobMgr.marshallResultObject(result)); } + @Override + public boolean updateVmNic(VirtualMachine vm, Nic nic, Boolean enabled) { + Outcome outcome = updateVmNicThroughJobQueue(vm, nic, enabled); + + retrieveVmFromJobOutcome(outcome, vm.getUuid(), "updateVmNic"); + + try { + Object jobResult = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); + if (jobResult instanceof Boolean) { + return BooleanUtils.isTrue((Boolean) jobResult); + } + } catch (ResourceUnavailableException | InsufficientCapacityException ex) { + throw new CloudRuntimeException(String.format("Exception while updating VM [%s] NIC. Check the logs for more information.", vm.getUuid())); + } + throw new CloudRuntimeException("Unexpected job execution result."); + } + + private boolean orchestrateUpdateVmNic(final VirtualMachine vm, final Nic nic, final Boolean enabled) throws ResourceUnavailableException { + if (vm.getState() == State.Running) { + try { + UpdateVmNicCommand updateVmNicCmd = new UpdateVmNicCommand(nic.getMacAddress(), vm.getName(), enabled); + Commands cmds = new Commands(Command.OnError.Stop); + cmds.addCommand("updatevmnic", updateVmNicCmd); + + _agentMgr.send(vm.getHostId(), cmds); + + UpdateVmNicAnswer updateVmNicAnswer = cmds.getAnswer(UpdateVmNicAnswer.class); + if (updateVmNicAnswer == null || !updateVmNicAnswer.getResult()) { + logger.warn("Unable to update VM %s NIC [{}].", vm.getName(), nic.getUuid()); + return false; + } + } catch (final OperationTimedoutException e) { + throw new AgentUnavailableException(String.format("Unable to update NIC %s for VM %s.", nic.getUuid(), vm.getUuid()), vm.getHostId(), e); + } + } + + NicVO nicVo = _nicsDao.findById(nic.getId()); + nicVo.setEnabled(enabled); + _nicsDao.persist(nicVo); + + return true; + } + + public Outcome updateVmNicThroughJobQueue(final VirtualMachine vm, final Nic nic, final Boolean isNicEnabled) { + Long vmId = vm.getId(); + String commandName = VmWorkUpdateNic.class.getName(); + Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); + + VmWorkJobVO workJob = pendingWorkJob.first(); + + if (workJob == null) { + Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); + + workJob = newVmWorkJobAndInfo.first(); + VmWorkUpdateNic workInfo = new VmWorkUpdateNic(newVmWorkJobAndInfo.second(), nic.getId(), isNicEnabled); + + setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); + } + AsyncJobExecutionContext.getCurrentExecutionContext().joinJob(workJob.getId()); + + return new VmJobVirtualMachineOutcome(workJob, vmId); + } + + @ReflectionUse + private Pair orchestrateUpdateVmNic(final VmWorkUpdateNic work) throws Exception { + VMInstanceVO vm = findVmById(work.getVmId()); + final NicVO nic = _entityMgr.findById(NicVO.class, work.getNicId()); + if (nic == null) { + throw new CloudRuntimeException(String.format("Unable to find NIC with ID %s.", work.getNicId())); + } + final boolean result = orchestrateUpdateVmNic(vm, nic, work.isEnabled()); + return new Pair<>(JobInfo.Status.SUCCEEDED, _jobMgr.marshallResultObject(result)); + } + private Pair findClusterAndHostIdForVmFromVolumes(long vmId) { Long clusterId = null; Long hostId = null; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateNic.java b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateNic.java new file mode 100644 index 00000000000..1c63cf34a19 --- /dev/null +++ b/engine/orchestration/src/main/java/com/cloud/vm/VmWorkUpdateNic.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.vm; + +public class VmWorkUpdateNic extends VmWork { + private static final long serialVersionUID = -8957066627929113278L; + + Long nicId; + Boolean enabled; + + public VmWorkUpdateNic(VmWork vmWork, Long nicId, Boolean enabled) { + super(vmWork); + this.nicId = nicId; + this.enabled = enabled; + } + + public Long getNicId() { + return nicId; + } + + public Boolean isEnabled() { + return enabled; + } +} diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index b6f31414655..2ae228adba9 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -58,6 +58,7 @@ import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.cloudstack.network.dao.NetworkPermissionDao; +import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; @@ -86,6 +87,7 @@ import com.cloud.api.query.dao.DomainRouterJoinDao; import com.cloud.api.query.vo.DomainRouterJoinVO; import com.cloud.bgp.BGPService; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.ASNumberVO; import com.cloud.dc.ClusterVO; @@ -214,6 +216,7 @@ import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingDetailsDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; import com.cloud.resource.ResourceManager; +import com.cloud.resourcelimit.CheckedReservation; import com.cloud.server.ManagementServer; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; @@ -447,6 +450,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra ClusterDao clusterDao; @Inject RoutedIpv4Manager routedIpv4Manager; + @Inject + private ReservationDao reservationDao; protected StateMachine2 _stateMachine; ScheduledExecutorService _executor; @@ -2326,7 +2331,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra for (final NetworkElement element : networkElements) { if (providersToImplement.contains(element.getProvider())) { if (!_networkModel.isProviderEnabledInPhysicalNetwork(_networkModel.getPhysicalNetworkId(network), element.getProvider().getName())) { - throw new CloudRuntimeException(String.format("Service provider %s either doesn't exist or is not enabled in physical network: %s", element.getProvider().getName(), _physicalNetworkDao.findById(network.getPhysicalNetworkId()))); + throw new CloudRuntimeException(String.format("Service provider %s either doesn't exist or is not enabled in physical network: %s", + element.getProvider().getName(), _physicalNetworkDao.findById(network.getPhysicalNetworkId()))); } if (element instanceof NetworkMigrationResponder) { if (!((NetworkMigrationResponder) element).prepareMigration(profile, network, vm, dest, context)) { @@ -2633,6 +2639,10 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkGuru guru = AdapterBase.getAdapterByName(networkGurus, network.getGuruName()); guru.deallocate(network, profile, vm); + if (nic.getReservationStrategy() == Nic.ReservationStrategy.Create) { + applyProfileToNicForRelease(nic, profile); + _nicDao.update(nic.getId(), nic); + } if (BooleanUtils.isNotTrue(preserveNics)) { _nicDao.remove(nic.getId()); } @@ -2747,12 +2757,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return null; } - final boolean updateResourceCount = resourceCountNeedsUpdate(ntwkOff, aclType); - //check resource limits - if (updateResourceCount) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.network, isDisplayNetworkEnabled); - } - // Validate network offering if (ntwkOff.getState() != NetworkOffering.State.Enabled) { // see NetworkOfferingVO @@ -2771,6 +2775,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra boolean ipv6 = false; + try (CheckedReservation networkReservation = new CheckedReservation(owner, domainId, Resource.ResourceType.network, null, null, 1L, reservationDao, _resourceLimitMgr)) { + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { ipv6 = true; } @@ -3110,8 +3116,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } - if (updateResourceCount) { - _resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.network, isDisplayNetworkEnabled); + if (isResourceCountUpdateNeeded(ntwkOff)) { + changeAccountResourceCountOrRecalculateDomainResourceCount(owner.getAccountId(), domainId, isDisplayNetworkEnabled, true); } UsageEventUtils.publishNetworkCreation(network); @@ -3122,6 +3128,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra CallContext.current().setEventDetails("Network ID: " + network.getUuid()); CallContext.current().putContextParameter(Network.class, network.getUuid()); return network; + } } @Override @@ -3487,9 +3494,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } final NetworkOffering ntwkOff = _entityMgr.findById(NetworkOffering.class, networkFinal.getNetworkOfferingId()); - final boolean updateResourceCount = resourceCountNeedsUpdate(ntwkOff, networkFinal.getAclType()); - if (updateResourceCount) { - _resourceLimitMgr.decrementResourceCount(networkFinal.getAccountId(), ResourceType.network, networkFinal.getDisplayNetwork()); + if (isResourceCountUpdateNeeded(ntwkOff)) { + changeAccountResourceCountOrRecalculateDomainResourceCount(networkFinal.getAccountId(), networkFinal.getDomainId(), networkFinal.getDisplayNetwork(), false); } } return deletedVlans.second(); @@ -3512,6 +3518,23 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return success; } + /** + * If it is a shared network with {@link ACLType#Domain}, it will belong to account {@link Account#ACCOUNT_ID_SYSTEM} and the resources will be not incremented for the + * domain. Therefore, we force the recalculation of the domain's resource count in this case. Otherwise, it will change the count for the account owner. + * @param incrementAccountResourceCount If true, the account resource count will be incremented by 1; otherwise, it will decremented by 1. + */ + private void changeAccountResourceCountOrRecalculateDomainResourceCount(Long accountId, Long domainId, boolean displayNetwork, boolean incrementAccountResourceCount) { + if (Account.ACCOUNT_ID_SYSTEM == accountId && ObjectUtils.isNotEmpty(domainId)) { + _resourceLimitMgr.recalculateDomainResourceCount(domainId, ResourceType.network, null); + } else { + if (incrementAccountResourceCount) { + _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.network, displayNetwork); + } else { + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.network, displayNetwork); + } + } + } + private void publishDeletedVlanRanges(List deletedVlanRangeToPublish) { if (CollectionUtils.isNotEmpty(deletedVlanRangeToPublish)) { for (VlanVO vlan : deletedVlanRangeToPublish) { @@ -3521,10 +3544,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } @Override - public boolean resourceCountNeedsUpdate(final NetworkOffering ntwkOff, final ACLType aclType) { - //Update resource count only for Isolated account specific non-system networks - final boolean updateResourceCount = ntwkOff.getGuestType() == GuestType.Isolated && !ntwkOff.isSystemOnly() && aclType == ACLType.Account; - return updateResourceCount; + public boolean isResourceCountUpdateNeeded(NetworkOffering networkOffering) { + return !networkOffering.isSystemOnly(); } protected Pair> deleteVlansInNetwork(final NetworkVO network, final long userId, final Account callerAccount) { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index e8c75afa81c..bf3985d3ce7 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -40,6 +40,7 @@ import javax.naming.ConfigurationException; import com.cloud.deploy.DeploymentClusterPlanner; import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ReservationHelper; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; @@ -82,6 +83,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.resourcelimit.Reserver; import org.apache.cloudstack.secret.PassphraseVO; import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.snapshot.SnapshotHelper; @@ -862,7 +864,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", create = true) @Override public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId) { + Long deviceId, boolean incrementResourceCount) { if (size == null) { size = offering.getDiskSize(); } else { @@ -901,7 +903,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati saveVolumeDetails(offering.getId(), vol.getId()); // Save usage event and update resource count for user vm volumes - if (vm.getType() == VirtualMachine.Type.User) { + if (vm.getType() == VirtualMachine.Type.User && incrementResourceCount) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size, Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); @@ -1938,14 +1940,20 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati template == null ? null : template.getSize(), vol.getPassphraseId() != null); - if (newSize != vol.getSize()) { - DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + if (newSize == vol.getSize()) { + return; + } + + DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + + List reservations = new ArrayList<>(); + try { VMInstanceVO vm = vol.getInstanceId() != null ? vmInstanceDao.findById(vol.getInstanceId()) : null; if (vm == null || vm.getType() == VirtualMachine.Type.User) { // Update resource count for user vm volumes when volume is attached if (newSize > vol.getSize()) { _resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()), - vol.isDisplay(), newSize - vol.getSize(), diskOffering); + vol.isDisplay(), newSize - vol.getSize(), diskOffering, reservations); _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), newSize - vol.getSize(), diskOffering); } else { @@ -1953,9 +1961,11 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vol.getSize() - newSize, diskOffering); } } - vol.setSize(newSize); - _volsDao.persist(vol); + } finally { + ReservationHelper.closeAll(reservations); } + vol.setSize(newSize); + _volsDao.persist(vol); } @Override diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java index 01afd0780f7..e4047cf7973 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java @@ -27,6 +27,6 @@ public interface AccountVlanMapDao extends GenericDao { public List listAccountVlanMapsByVlan(long vlanDbId); - public AccountVlanMapVO findAccountVlanMap(long accountId, long vlanDbId); + public AccountVlanMapVO findAccountVlanMap(Long accountId, long vlanDbId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java index 12114770f11..0844bb77caa 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java @@ -48,9 +48,9 @@ public class AccountVlanMapDaoImpl extends GenericDaoBase sc = AccountVlanSearch.create(); - sc.setParameters("accountId", accountId); + sc.setParametersIfNotNull("accountId", accountId); sc.setParameters("vlanDbId", vlanDbId); return findOneIncludingRemovedBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java index 6af16bbace9..d14ccbe86ca 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java @@ -24,5 +24,5 @@ import com.cloud.utils.db.GenericDao; public interface DomainVlanMapDao extends GenericDao { public List listDomainVlanMapsByDomain(long domainId); public List listDomainVlanMapsByVlan(long vlanDbId); - public DomainVlanMapVO findDomainVlanMap(long domainId, long vlanDbId); + public DomainVlanMapVO findDomainVlanMap(Long domainId, long vlanDbId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java index f789721d5fd..0b4c781349f 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java @@ -46,9 +46,9 @@ public class DomainVlanMapDaoImpl extends GenericDaoBase } @Override - public DomainVlanMapVO findDomainVlanMap(long domainId, long vlanDbId) { + public DomainVlanMapVO findDomainVlanMap(Long domainId, long vlanDbId) { SearchCriteria sc = DomainVlanSearch.create(); - sc.setParameters("domainId", domainId); + sc.setParametersIfNotNull("domainId", domainId); sc.setParameters("vlanDbId", vlanDbId); return findOneIncludingRemovedBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java index 56d971bbe01..1afa0d22dcc 100644 --- a/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/domain/dao/DomainDaoImpl.java @@ -262,7 +262,7 @@ public class DomainDaoImpl extends GenericDaoBase implements Dom SearchCriteria sc = DomainPairSearch.create(); sc.setParameters("id", parentId, childId); - List domainPair = listBy(sc); + List domainPair = listIncludingRemovedBy(sc); if ((domainPair != null) && (domainPair.size() == 2)) { DomainVO d1 = domainPair.get(0); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 7a00829fd44..0d86ca0e48c 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao { HostTagResponse newHostTagResponse(HostTagVO hostTag); List searchByIds(Long... hostTagIds); + + /** + * List all host tags defined on hosts within a cluster + */ + List listByClusterId(Long clusterId); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index 4aa14a31cfc..d3fee6a2676 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase implements private final SearchBuilder stSearch; private final SearchBuilder tagIdsearch; private final SearchBuilder ImplicitTagsSearch; + private final GenericSearchBuilder tagSearch; @Inject private ConfigurationDao _configDao; + @Inject + private HostDao hostDao; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); @@ -72,6 +76,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ); ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); ImplicitTagsSearch.done(); + + tagSearch = createSearchBuilder(String.class); + tagSearch.selectFields(tagSearch.entity().getTag()); + tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN); + tagSearch.done(); } @Override @@ -235,4 +244,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements return tagList; } + + @Override + public List listByClusterId(Long clusterId) { + List hostIds = hostDao.listIdsByClusterId(clusterId); + if (CollectionUtils.isEmpty(hostIds)) { + return new ArrayList<>(); + } + SearchCriteria sc = tagSearch.create(); + sc.setParameters("hostIdIN", hostIds.toArray()); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 4e8b6204f72..9f7ffabac93 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -193,6 +193,7 @@ public class NetworkDaoImpl extends GenericDaoBaseimplements Ne PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ); PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN); PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ); + PersistentNetworkSearch.and("dc", PersistentNetworkSearch.entity().getDataCenterId(), Op.EQ); PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL); final SearchBuilder persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder(); persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ); diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java index 1a2b098c40a..f24f3f3b67c 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDao.java @@ -35,7 +35,7 @@ public interface GuestOSDao extends GenericDao { List listByDisplayName(String displayName); - Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay); + Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, List ids, Long osCategoryId, String description, String keyword, Boolean forDisplay); List listIdsByCategoryId(final long categoryId); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java index 881be207c1a..09f8772fb59 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/GuestOSDaoImpl.java @@ -125,12 +125,12 @@ public class GuestOSDaoImpl extends GenericDaoBase implements G return listBy(sc); } - public Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, Long id, Long osCategoryId, String description, String keyword, Boolean forDisplay) { + public Pair, Integer> listGuestOSByCriteria(Long startIndex, Long pageSize, List ids, Long osCategoryId, String description, String keyword, Boolean forDisplay) { final Filter searchFilter = new Filter(GuestOSVO.class, "displayName", true, startIndex, pageSize); final SearchCriteria sc = createSearchCriteria(); - if (id != null) { - sc.addAnd("id", SearchCriteria.Op.EQ, id); + if (CollectionUtils.isNotEmpty(ids)) { + sc.addAnd("id", SearchCriteria.Op.IN, ids.toArray()); } if (osCategoryId != null) { diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java index 80e1b7d4d4b..f167b573187 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -170,6 +170,7 @@ public class SnapshotDaoImpl extends GenericDaoBase implements CountSnapshotsByAccount.select(null, Func.COUNT, null); CountSnapshotsByAccount.and("account", CountSnapshotsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); CountSnapshotsByAccount.and("status", CountSnapshotsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountSnapshotsByAccount.and("snapshotTypeNEQ", CountSnapshotsByAccount.entity().getSnapshotType(), SearchCriteria.Op.NIN); CountSnapshotsByAccount.and("removed", CountSnapshotsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CountSnapshotsByAccount.done(); @@ -220,6 +221,7 @@ public class SnapshotDaoImpl extends GenericDaoBase implements SearchCriteria sc = CountSnapshotsByAccount.create(); sc.setParameters("account", accountId); sc.setParameters("status", State.Error, State.Destroyed); + sc.setParameters("snapshotTypeNEQ", Snapshot.Type.GROUP.ordinal()); return customSearch(sc, null).get(0); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index a03b94faa79..717e3e782f2 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -166,4 +166,12 @@ public interface VolumeDao extends GenericDao, StateDao implements Vol private final SearchBuilder storeAndInstallPathSearch; private final SearchBuilder volumeIdSearch; protected GenericSearchBuilder CountByAccount; + protected final SearchBuilder ExternalUuidSearch; protected GenericSearchBuilder primaryStorageSearch; protected GenericSearchBuilder primaryStorageSearch2; protected GenericSearchBuilder secondaryStorageSearch; @@ -459,6 +460,10 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol CountByAccount.and("idNIN", CountByAccount.entity().getId(), Op.NIN); CountByAccount.done(); + ExternalUuidSearch = createSearchBuilder(); + ExternalUuidSearch.and("externalUuid", ExternalUuidSearch.entity().getExternalUuid(), Op.EQ); + ExternalUuidSearch.done(); + primaryStorageSearch = createSearchBuilder(SumCount.class); primaryStorageSearch.select("sum", Func.SUM, primaryStorageSearch.entity().getSize()); primaryStorageSearch.and("accountId", primaryStorageSearch.entity().getAccountId(), Op.EQ); @@ -934,4 +939,11 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol sc.and(sc.entity().getState(), SearchCriteria.Op.IN, (Object[]) states); return sc.find(); } + + @Override + public VolumeVO findByExternalUuid(String externalUuid) { + SearchCriteria sc = ExternalUuidSearch.create(); + sc.setParameters("externalUuid", externalUuid); + return findOneBy(sc); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java index c9610f7b9ff..813351d7534 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42200to42210.java @@ -16,6 +16,14 @@ // under the License. package com.cloud.upgrade.dao; +import org.apache.cloudstack.vm.UnmanagedVMsManager; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; + public class Upgrade42200to42210 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @Override @@ -27,4 +35,47 @@ public class Upgrade42200to42210 extends DbUpgradeAbstractImpl implements DbUpgr public String getUpgradedVersion() { return "4.22.1.0"; } + + @Override + public void performDataMigration(Connection conn) { + removeDuplicateKVMImportTemplates(conn); + } + + private void removeDuplicateKVMImportTemplates(Connection conn) { + List templateIds = new ArrayList<>(); + try (PreparedStatement selectStmt = conn.prepareStatement(String.format("SELECT id FROM cloud.vm_template WHERE name='%s' ORDER BY id ASC", UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME))) { + ResultSet rs = selectStmt.executeQuery(); + while (rs.next()) { + templateIds.add(rs.getLong(1)); + } + + if (templateIds.size() <= 1) { + return; + } + + logger.info("Removing duplicate template " + UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME + " entries"); + Long firstTemplateId = templateIds.get(0); + + String updateTemplateSql = "UPDATE cloud.vm_instance SET vm_template_id = ? WHERE vm_template_id = ?"; + String deleteTemplateSql = "DELETE FROM cloud.vm_template WHERE id = ?"; + + try (PreparedStatement updateTemplateStmt = conn.prepareStatement(updateTemplateSql); + PreparedStatement deleteTemplateStmt = conn.prepareStatement(deleteTemplateSql)) { + for (int i = 1; i < templateIds.size(); i++) { + Long duplicateTemplateId = templateIds.get(i); + + // Update VM references + updateTemplateStmt.setLong(1, firstTemplateId); + updateTemplateStmt.setLong(2, duplicateTemplateId); + updateTemplateStmt.executeUpdate(); + + // Delete duplicate dummy template + deleteTemplateStmt.setLong(1, duplicateTemplateId); + deleteTemplateStmt.executeUpdate(); + } + } + } catch (Exception e) { + logger.warn("Failed to remove duplicate template " + UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME + " entries", e); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java index df4743894c9..393f2039950 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java @@ -17,7 +17,12 @@ package com.cloud.upgrade.dao; import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @@ -42,4 +47,46 @@ public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgr return new InputStream[] {script}; } + + @Override + public void performDataMigration(Connection conn) { + unhideJsInterpretationEnabled(conn); + } + + protected void unhideJsInterpretationEnabled(Connection conn) { + String value = getJsInterpretationEnabled(conn); + if (value != null) { + updateJsInterpretationEnabledFields(conn, value); + } + } + + protected String getJsInterpretationEnabled(Connection conn) { + String query = "SELECT value FROM cloud.configuration WHERE name = 'js.interpretation.enabled' AND category = 'Hidden';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getString("value"); + } + logger.debug("Unable to retrieve value of hidden configuration 'js.interpretation.enabled'. The configuration may already be unhidden."); + return null; + } catch (SQLException e) { + throw new CloudRuntimeException("Error while retrieving value of hidden configuration 'js.interpretation.enabled'.", e); + } + } + + protected void updateJsInterpretationEnabledFields(Connection conn, String encryptedValue) { + String query = "UPDATE cloud.configuration SET value = ?, category = 'System', component = 'JsInterpreter', is_dynamic = 1 WHERE name = 'js.interpretation.enabled';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + String decryptedValue = DBEncryptionUtil.decrypt(encryptedValue); + logger.info("Updating setting 'js.interpretation.enabled' to decrypted value [{}], category 'System', component 'JsInterpreter', and is_dynamic '1'.", decryptedValue); + pstmt.setString(1, decryptedValue); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Error while unhiding configuration 'js.interpretation.enabled'.", e); + } catch (CloudRuntimeException e) { + logger.warn("Error while decrypting configuration 'js.interpretation.enabled'. The configuration may already be decrypted."); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/NicVO.java b/engine/schema/src/main/java/com/cloud/vm/NicVO.java index 6c569e22dd9..65946b8d821 100644 --- a/engine/schema/src/main/java/com/cloud/vm/NicVO.java +++ b/engine/schema/src/main/java/com/cloud/vm/NicVO.java @@ -131,6 +131,9 @@ public class NicVO implements Nic { @Column(name = "mtu") Integer mtu; + @Column(name = "enabled") + boolean enabled; + @Transient transient String nsxLogicalSwitchUuid; @@ -143,6 +146,7 @@ public class NicVO implements Nic { this.networkId = configurationId; this.state = State.Allocated; this.vmType = vmType; + this.enabled = true; } @Override @@ -397,6 +401,14 @@ public class NicVO implements Nic { this.nsxLogicalSwitchPortUuid = nsxLogicalSwitchPortUuid; } + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + @Override public int hashCode() { return new HashCodeBuilder(17, 31).append(id).toHashCode(); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 23541c2431e..4fd3e729e0d 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -21,6 +21,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.Pair; @@ -192,4 +193,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< int getVmCountByOfferingNotInDomain(Long serviceOfferingId, List domainIds); List listByIdsIncludingRemoved(List ids); + + List listDeleteProtectedVmsByAccountId(long accountId); + + List listDeleteProtectedVmsByDomainIds(Set domainIds); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 589a63ea0d8..6ffa7cd5962 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -25,11 +25,13 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -106,6 +108,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder IdsPowerStateSelectSearch; GenericSearchBuilder CountByOfferingId; GenericSearchBuilder CountUserVmNotInDomain; + SearchBuilder DeleteProtectedVmSearchByAccount; + SearchBuilder DeleteProtectedVmSearchByDomainIds; @Inject ResourceTagDao tagsDao; @@ -368,6 +372,19 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem CountUserVmNotInDomain.and("domainIdsNotIn", CountUserVmNotInDomain.entity().getDomainId(), Op.NIN); CountUserVmNotInDomain.done(); + DeleteProtectedVmSearchByAccount = createSearchBuilder(); + DeleteProtectedVmSearchByAccount.selectFields(DeleteProtectedVmSearchByAccount.entity().getUuid()); + DeleteProtectedVmSearchByAccount.and(ApiConstants.ACCOUNT_ID, DeleteProtectedVmSearchByAccount.entity().getAccountId(), Op.EQ); + DeleteProtectedVmSearchByAccount.and(ApiConstants.DELETE_PROTECTION, DeleteProtectedVmSearchByAccount.entity().isDeleteProtection(), Op.EQ); + DeleteProtectedVmSearchByAccount.and(ApiConstants.REMOVED, DeleteProtectedVmSearchByAccount.entity().getRemoved(), Op.NULL); + DeleteProtectedVmSearchByAccount.done(); + + DeleteProtectedVmSearchByDomainIds = createSearchBuilder(); + DeleteProtectedVmSearchByDomainIds.selectFields(DeleteProtectedVmSearchByDomainIds.entity().getUuid()); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.DOMAIN_IDS, DeleteProtectedVmSearchByDomainIds.entity().getDomainId(), Op.IN); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.DELETE_PROTECTION, DeleteProtectedVmSearchByDomainIds.entity().isDeleteProtection(), Op.EQ); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.REMOVED, DeleteProtectedVmSearchByDomainIds.entity().getRemoved(), Op.NULL); + DeleteProtectedVmSearchByDomainIds.done(); } @Override @@ -1296,4 +1313,22 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem sc.setParameters("ids", ids.toArray()); return listIncludingRemovedBy(sc); } + + @Override + public List listDeleteProtectedVmsByAccountId(long accountId) { + SearchCriteria sc = DeleteProtectedVmSearchByAccount.create(); + sc.setParameters(ApiConstants.ACCOUNT_ID, accountId); + sc.setParameters(ApiConstants.DELETE_PROTECTION, true); + Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); + return listBy(sc, filter); + } + + @Override + public List listDeleteProtectedVmsByDomainIds(Set domainIds) { + SearchCriteria sc = DeleteProtectedVmSearchByDomainIds.create(); + sc.setParameters(ApiConstants.DOMAIN_IDS, domainIds.toArray()); + sc.setParameters(ApiConstants.DELETE_PROTECTION, true); + Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); + return listBy(sc, filter); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java index ee1783a9c89..87b7dab1ff7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java @@ -21,20 +21,14 @@ import java.util.Date; import java.util.List; import com.cloud.utils.DateUtil; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.BackupScheduleVO; import com.cloud.utils.db.GenericDao; public interface BackupScheduleDao extends GenericDao { - BackupScheduleVO findByVM(Long vmId); - List listByVM(Long vmId); BackupScheduleVO findByVMAndIntervalType(Long vmId, DateUtil.IntervalType intervalType); List getSchedulesToExecute(Date currentTimestamp); - - BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index d9cf7b63680..972af73391a 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -17,28 +17,23 @@ package org.apache.cloudstack.backup.dao; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.Date; import java.util.List; import javax.annotation.PostConstruct; -import javax.inject.Inject; import com.cloud.utils.DateUtil; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.backup.BackupSchedule; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.TransactionLegacy; import org.apache.cloudstack.backup.BackupScheduleVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.dao.VMInstanceDao; public class BackupScheduleDaoImpl extends GenericDaoBase implements BackupScheduleDao { - - @Inject - VMInstanceDao vmInstanceDao; - private SearchBuilder backupScheduleSearch; private SearchBuilder executableSchedulesSearch; @@ -59,13 +54,6 @@ public class BackupScheduleDaoImpl extends GenericDaoBase sc = backupScheduleSearch.create(); - sc.setParameters("vm_id", vmId); - return findOneBy(sc); - } - @Override public List listByVM(Long vmId) { SearchCriteria sc = backupScheduleSearch.create(); @@ -88,21 +76,19 @@ public class BackupScheduleDaoImpl extends GenericDaoBase listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State... state); + List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId); SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index cdf903407c1..8b7a2b78de7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -67,7 +67,8 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStoreRoleEqStateNeqRefCntNeq; protected SearchBuilder searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; - private SearchBuilder idStateNeqSearch; + private SearchBuilder idStateNinSearch; + private SearchBuilder idEqRoleEqStateInSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; @@ -146,10 +147,15 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, State... state) { + SearchCriteria sc = idEqRoleEqStateInSearch.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STORE_ROLE, role); + sc.setParameters(STATE, (Object[])state); + return listBy(sc); + } + @Override public SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId) { try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) { @@ -480,7 +495,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotIdWithNonDestroyedState(long snapshotId) { - SearchCriteria sc = idStateNeqSearch.create(); + SearchCriteria sc = idStateNinSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed.name()); return listBy(sc); @@ -488,7 +503,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase findBySnapshotIdAndNotInDestroyedHiddenState(long snapshotId) { - SearchCriteria sc = idStateNeqSearch.create(); + SearchCriteria sc = idStateNinSearch.create(); sc.setParameters(SNAPSHOT_ID, snapshotId); sc.setParameters(STATE, State.Destroyed.name(), State.Hidden.name()); return listBy(sc); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4c65f37e0fe..000b54b7207 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -687,22 +687,23 @@ CREATE TABLE IF NOT EXISTS `cloud`.`backup_details` ( UPDATE `cloud`.`backups` b INNER JOIN `cloud`.`vm_instance` vm ON b.vm_id = vm.id SET b.backed_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id @@ -711,22 +712,23 @@ SET b.backed_volumes = ( -- Add diskOfferingId, deviceId, minIops and maxIops to backup_volumes in vm_instance table UPDATE `cloud`.`vm_instance` vm SET vm.backup_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index a8a3d3f7bd4..43e0d2c78a9 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -33,3 +33,23 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; -- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; + +-- Sanitize legacy network-level addressing fields for Public networks +UPDATE `cloud`.`networks` +SET `broadcast_uri` = NULL, + `gateway` = NULL, + `cidr` = NULL, + `ip6_gateway` = NULL, + `ip6_cidr` = NULL +WHERE `traffic_type` = 'Public'; + +UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template'; + +-- Update existing vm_template records with NULL type to "USER" +UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL; + +-- remove unused config item +DELETE FROM `cloud`.`configuration` WHERE name = 'consoleproxy.cmd.port'; + +-- Drops the unused "backup_interval_type" column of the "cloud.backups" table +ALTER TABLE `cloud`.`backups` DROP COLUMN `backup_interval_type`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 2e50fc0a1fe..76669903877 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -115,5 +115,8 @@ CALL `cloud`.`IDEMPOTENT_UPDATE_API_PERMISSION`('Resource Admin', 'deleteUserKey -- Add conserve mode for VPC offerings CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tinyint(1) unsigned NULL DEFAULT 0 COMMENT ''True if the VPC offering is IP conserve mode enabled, allowing public IP services to be used across multiple VPC tiers'' '); +--- Disable/enable NICs +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); + -- Add description for secondary IP addresses CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nic_secondary_ips','description', 'varchar(2048) DEFAULT NULL'); \ No newline at end of file diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql index a12e02bcfdb..d5f17606cb4 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.domain_router_view.sql @@ -76,6 +76,7 @@ select nics.broadcast_uri broadcast_uri, nics.isolation_uri isolation_uri, nics.mtu mtu, + nics.enabled is_nic_enabled, vpc.id vpc_id, vpc.uuid vpc_uuid, vpc.name vpc_name, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 94bc8640fd5..c169cbce217 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -79,6 +79,7 @@ SELECT `vm_template`.`format` AS `template_format`, `vm_template`.`display_text` AS `template_display_text`, `vm_template`.`enable_password` AS `password_enabled`, + `vm_template`.`extension_id` AS `template_extension_id`, `iso`.`id` AS `iso_id`, `iso`.`uuid` AS `iso_uuid`, `iso`.`name` AS `iso_name`, @@ -141,6 +142,7 @@ SELECT `nics`.`mac_address` AS `mac_address`, `nics`.`broadcast_uri` AS `broadcast_uri`, `nics`.`isolation_uri` AS `isolation_uri`, + `nics`.`enabled` AS `is_nic_enabled`, `vpc`.`id` AS `vpc_id`, `vpc`.`uuid` AS `vpc_uuid`, `networks`.`uuid` AS `network_uuid`, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql new file mode 100644 index 00000000000..0646c3b6f91 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_summary_view.sql @@ -0,0 +1,48 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- cloud_usage.quota_summary_view source + +-- Create view for quota summary +DROP VIEW IF EXISTS `cloud_usage`.`quota_summary_view`; +CREATE VIEW `cloud_usage`.`quota_summary_view` AS +SELECT + cloud_usage.quota_account.account_id AS account_id, + cloud_usage.quota_account.quota_balance AS quota_balance, + cloud_usage.quota_account.quota_balance_date AS quota_balance_date, + cloud_usage.quota_account.quota_enforce AS quota_enforce, + cloud_usage.quota_account.quota_min_balance AS quota_min_balance, + cloud_usage.quota_account.quota_alert_date AS quota_alert_date, + cloud_usage.quota_account.quota_alert_type AS quota_alert_type, + cloud_usage.quota_account.last_statement_date AS last_statement_date, + cloud.account.uuid AS account_uuid, + cloud.account.account_name AS account_name, + cloud.account.state AS account_state, + cloud.account.removed AS account_removed, + cloud.domain.id AS domain_id, + cloud.domain.uuid AS domain_uuid, + cloud.domain.name AS domain_name, + cloud.domain.path AS domain_path, + cloud.domain.removed AS domain_removed, + cloud.projects.uuid AS project_uuid, + cloud.projects.name AS project_name, + cloud.projects.removed AS project_removed +FROM + cloud_usage.quota_account + INNER JOIN cloud.account ON (cloud.account.id = cloud_usage.quota_account.account_id) + INNER JOIN cloud.domain ON (cloud.domain.id = cloud.account.domain_id) + LEFT JOIN cloud.projects ON (cloud.account.type = 5 AND cloud.account.id = cloud.projects.project_account_id); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java index 275f41de43a..bcade3a371c 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/StorageSystemDataMotionStrategy.java @@ -148,8 +148,8 @@ import java.util.HashSet; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; -import static org.apache.cloudstack.vm.UnmanagedVMsManagerImpl.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME; -import static org.apache.cloudstack.vm.UnmanagedVMsManagerImpl.VM_IMPORT_DEFAULT_TEMPLATE_NAME; +import static org.apache.cloudstack.vm.UnmanagedVMsManager.KVM_VM_IMPORT_DEFAULT_TEMPLATE_NAME; +import static org.apache.cloudstack.vm.UnmanagedVMsManager.VM_IMPORT_DEFAULT_TEMPLATE_NAME; public class StorageSystemDataMotionStrategy implements DataMotionStrategy { protected Logger logger = LogManager.getLogger(getClass()); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java index 7174336113b..88f479c0904 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java @@ -120,7 +120,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { private final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error, Snapshot.State.Hidden); public SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { - List snaps = snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + List snaps = snapshotStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(snapshotId, DataStoreRole.Image, State.Ready, State.Hidden); for (SnapshotDataStoreVO ref : snaps) { if (zoneId == dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { return ref; diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java index 7199fce1d34..aced750bd32 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.datastore.api.Volume; import com.cloud.agent.api.VMSnapshotTO; import com.cloud.alert.AlertManager; @@ -200,11 +201,35 @@ public class ScaleIOVMSnapshotStrategy extends ManagerBase implements VMSnapshot if (volumeIds != null && !volumeIds.isEmpty()) { List vmSnapshotDetails = new ArrayList(); vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "SnapshotGroupId", snapshotGroupId, false)); + Map snapshotNameToSrcPathMap = new HashMap<>(); + for (Map.Entry entry : srcVolumeDestSnapshotMap.entrySet()) { + snapshotNameToSrcPathMap.put(entry.getValue(), entry.getKey()); + } - for (int index = 0; index < volumeIds.size(); index++) { - String volumeSnapshotName = srcVolumeDestSnapshotMap.get(ScaleIOUtil.getVolumePath(volumeTOs.get(index).getPath())); - String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(volumeIds.get(index), volumeSnapshotName); - vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "Vol_" + volumeTOs.get(index).getId() + "_Snapshot", pathWithScaleIOVolumeName, false)); + for (String snapshotVolumeId : volumeIds) { + // Use getVolume() to fetch snapshot volume details and get its name + Volume snapshotVolume = client.getVolume(snapshotVolumeId); + if (snapshotVolume == null) { + throw new CloudRuntimeException("Cannot find snapshot volume with id: " + snapshotVolumeId); + } + String snapshotName = snapshotVolume.getName(); + + // Match back to source volume path + String srcVolumePath = snapshotNameToSrcPathMap.get(snapshotName); + if (srcVolumePath == null) { + throw new CloudRuntimeException("Cannot match snapshot " + snapshotName + " to a source volume"); + } + + // Find the matching VolumeObjectTO by path + VolumeObjectTO matchedVolume = volumeTOs.stream() + .filter(v -> ScaleIOUtil.getVolumePath(v.getPath()).equals(srcVolumePath)) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException("Cannot find source volume for path: " + srcVolumePath)); + + String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, snapshotName); + vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), + "Vol_" + matchedVolume.getId() + "_Snapshot", + pathWithScaleIOVolumeName, false)); } vmSnapshotDetailsDao.saveDetails(vmSnapshotDetails); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java index e3f28a7012c..31b13fc279e 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java @@ -111,7 +111,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { FreezeThawVMAnswer freezeAnswer = null; FreezeThawVMCommand thawCmd = null; FreezeThawVMAnswer thawAnswer = null; - List forRollback = new ArrayList<>(); + List snapshotsForRollback = new ArrayList<>(); long startFreeze = 0; try { vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshotVO, VMSnapshot.Event.CreateRequested); @@ -165,7 +165,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { logger.info("The virtual machine is frozen"); for (VolumeInfo vol : vinfos) { long startSnapshtot = System.nanoTime(); - SnapshotInfo snapInfo = createDiskSnapshot(vmSnapshot, forRollback, vol); + SnapshotInfo snapInfo = createDiskSnapshot(vmSnapshot, snapshotsForRollback, vol); if (snapInfo == null) { thawAnswer = (FreezeThawVMAnswer) agentMgr.send(hostId, thawCmd); @@ -222,7 +222,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { } } if (!result) { - for (SnapshotInfo snapshotInfo : forRollback) { + for (SnapshotInfo snapshotInfo : snapshotsForRollback) { rollbackDiskSnapshot(snapshotInfo); } try { @@ -388,10 +388,16 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { //Rollback if one of disks snapshot fails protected void rollbackDiskSnapshot(SnapshotInfo snapshotInfo) { + if (snapshotInfo == null) { + return; + } Long snapshotID = snapshotInfo.getId(); SnapshotVO snapshot = snapshotDao.findById(snapshotID); + if (snapshot == null) { + return; + } deleteSnapshotByStrategy(snapshot); - logger.debug("Rollback is executed: deleting snapshot with id:" + snapshotID); + logger.debug("Rollback is executed: deleting snapshot with id: {}", snapshotID); } protected void deleteSnapshotByStrategy(SnapshotVO snapshot) { @@ -434,7 +440,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { } } - protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List forRollback, VolumeInfo vol) { + protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List snapshotsForRollback, VolumeInfo vol) { String snapshotName = vmSnapshot.getId() + "_" + vol.getUuid(); SnapshotVO snapshot = new SnapshotVO(vol.getDataCenterId(), vol.getAccountId(), vol.getDomainId(), vol.getId(), vol.getDiskOfferingId(), snapshotName, (short) Snapshot.Type.GROUP.ordinal(), Snapshot.Type.GROUP.name(), vol.getSize(), vol.getMinIops(), vol.getMaxIops(), Hypervisor.HypervisorType.KVM, null); @@ -448,6 +454,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { vol.addPayload(setPayload(vol, snapshot, quiescevm)); SnapshotInfo snapshotInfo = snapshotDataFactory.getSnapshot(snapshot.getId(), vol.getDataStore()); snapshotInfo.addPayload(vol.getpayload()); + snapshotsForRollback.add(snapshotInfo); SnapshotStrategy snapshotStrategy = storageStrategyFactory.getSnapshotStrategy(snapshotInfo, SnapshotOperation.TAKE); if (snapshotStrategy == null) { throw new CloudRuntimeException("Could not find strategy for snapshot uuid:" + snapshotInfo.getUuid()); @@ -455,8 +462,6 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { snapshotInfo = snapshotStrategy.takeSnapshot(snapshotInfo); if (snapshotInfo == null) { throw new CloudRuntimeException("Failed to create snapshot"); - } else { - forRollback.add(snapshotInfo); } vmSnapshotDetailsDao.persist(new VMSnapshotDetailsVO(vmSnapshot.getId(), STORAGE_SNAPSHOT, String.valueOf(snapshot.getId()), true)); snapshotInfo.markBackedUp(); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index 53f98c18f1b..41bfaa6f0c7 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -257,11 +257,6 @@ public class DefaultSnapshotStrategyTest { @Test public void testGetSnapshotImageStoreRefNull() { - SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); - Mockito.when(ref1.getDataStoreId()).thenReturn(1L); - Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); - Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); - Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(2L); Assert.assertNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); } @@ -270,7 +265,7 @@ public class DefaultSnapshotStrategyTest { SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); Mockito.when(ref1.getDataStoreId()).thenReturn(1L); Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); - Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + Mockito.when(snapshotDataStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.any(), Mockito.any())).thenReturn(List.of(ref1)); Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(1L); Assert.assertNotNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); } diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java index 050c0246aba..7d5d3c786e8 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java @@ -155,7 +155,7 @@ public class VMSnapshotStrategyKVMTest extends TestCase{ @Test public void testCreateDiskSnapshotBasedOnStrategy() throws Exception { VMSnapshotVO vmSnapshot = Mockito.mock(VMSnapshotVO.class); - List forRollback = new ArrayList<>(); + List snapshotsForRollback = new ArrayList<>(); VolumeInfo vol = Mockito.mock(VolumeInfo.class); SnapshotInfo snapshotInfo = Mockito.mock(SnapshotInfo.class); SnapshotStrategy strategy = Mockito.mock(SnapshotStrategy.class); @@ -179,7 +179,7 @@ public class VMSnapshotStrategyKVMTest extends TestCase{ VMSnapshotDetailsVO vmDetails = new VMSnapshotDetailsVO(vmSnapshot.getId(), volUuid, String.valueOf(snapshot.getId()), false); when(vmSnapshotDetailsDao.persist(any())).thenReturn(vmDetails); - info = vmStrategy.createDiskSnapshot(vmSnapshot, forRollback, vol); + info = vmStrategy.createDiskSnapshot(vmSnapshot, snapshotsForRollback, vol); assertNotNull(info); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index cd9ab3adb21..4057f7a051b 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -145,10 +145,10 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement storageType = "shared"; } - logger.debug(String.format( - "Filtering storage pools by capacity type [%s] as the first storage pool of the list, with name [%s] and ID [%s], is a [%s] storage.", + logger.info( + "Filtering storage pools by capacity type [{}] as the first storage pool of the list, with name [{}] and ID [{}], is a [{}] storage.", capacityType, storagePool.getName(), storagePool.getUuid(), storageType - )); + ); Pair, Map> result = capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType); List poolIdsByCapacity = result.first(); @@ -185,7 +185,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement Long clusterId = plan.getClusterId(); List poolIdsByVolCount = volumeDao.listPoolIdsByVolumeCount(dcId, podId, clusterId, account.getAccountId()); - logger.debug(String.format("List of pools in ascending order of number of volumes for account [%s] is [%s].", account, poolIdsByVolCount)); + logger.debug("List of pools in ascending order of number of volumes for account [{}] is [{}].", account, poolIdsByVolCount); // now filter the given list of Pools by this ordered list Map poolMap = new HashMap<>(); @@ -206,16 +206,11 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement @Override public List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan, DiskProfile dskCh) { - if (logger.isTraceEnabled()) { - logger.trace("reordering pools"); - } if (pools == null) { - logger.trace("There are no pools to reorder; returning null."); + logger.info("There are no pools to reorder."); return null; } - if (logger.isTraceEnabled()) { - logger.trace(String.format("reordering %d pools", pools.size())); - } + logger.info("Reordering [{}] pools", pools.size()); Account account = null; if (vmProfile.getVirtualMachine() != null) { account = vmProfile.getOwner(); @@ -224,9 +219,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement pools = reorderStoragePoolsBasedOnAlgorithm(pools, plan, account); if (vmProfile.getVirtualMachine() == null) { - if (logger.isTraceEnabled()) { - logger.trace("The VM is null, skipping pools reordering by disk provisioning type."); - } + logger.info("The VM is null, skipping pool reordering by disk provisioning type."); return pools; } @@ -240,14 +233,10 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement List reorderStoragePoolsBasedOnAlgorithm(List pools, DeploymentPlan plan, Account account) { String volumeAllocationAlgorithm = VolumeOrchestrationService.VolumeAllocationAlgorithm.value(); - logger.debug("Using volume allocation algorithm {} to reorder pools.", volumeAllocationAlgorithm); + logger.info("Using volume allocation algorithm {} to reorder pools.", volumeAllocationAlgorithm); if (volumeAllocationAlgorithm.equals("random") || (account == null)) { reorderRandomPools(pools); } else if (StringUtils.equalsAny(volumeAllocationAlgorithm, "userdispersing", "firstfitleastconsumed")) { - if (logger.isTraceEnabled()) { - logger.trace("Using reordering algorithm {}", volumeAllocationAlgorithm); - } - if (volumeAllocationAlgorithm.equals("userdispersing")) { pools = reorderPoolsByNumberOfVolumes(plan, pools, account); } else { @@ -259,16 +248,15 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement void reorderRandomPools(List pools) { StorageUtil.traceLogStoragePools(pools, logger, "pools to choose from: "); - if (logger.isTraceEnabled()) { - logger.trace("Shuffle this so that we don't check the pools in the same order. Algorithm == 'random' (or no account?)"); - } - StorageUtil.traceLogStoragePools(pools, logger, "pools to shuffle: "); + logger.trace("Shuffle this so that we don't check the pools in the same order. Algorithm == 'random' (or no account?)"); + logger.debug("Pools to shuffle: [{}]", pools); Collections.shuffle(pools, secureRandom); - StorageUtil.traceLogStoragePools(pools, logger, "shuffled list of pools to choose from: "); + logger.debug("Shuffled list of pools to choose from: [{}]", pools); } private List reorderPoolsByDiskProvisioningType(List pools, DiskProfile diskProfile) { if (diskProfile != null && diskProfile.getProvisioningType() != null && !diskProfile.getProvisioningType().equals(Storage.ProvisioningType.THIN)) { + logger.info("Reordering [{}] pools by disk provisioning type [{}].", pools.size(), diskProfile.getProvisioningType()); List reorderedPools = new ArrayList<>(); int preferredIndex = 0; for (StoragePool pool : pools) { @@ -282,22 +270,28 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement reorderedPools.add(preferredIndex++, pool); } } + logger.debug("Reordered list of pools by disk provisioning type [{}]: [{}]", diskProfile.getProvisioningType(), reorderedPools); return reorderedPools; } else { + if (diskProfile == null) { + logger.info("Reordering pools by disk provisioning type wasn't necessary, since no disk profile was found."); + } else { + logger.debug("Reordering pools by disk provisioning type wasn't necessary, since the provisioning type is [{}].", diskProfile.getProvisioningType()); + } return pools; } } protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, DeploymentPlan plan) { - logger.debug(String.format("Checking if storage pool [%s] is suitable to disk [%s].", pool, dskCh)); + logger.debug("Checking if storage pool [{}] is suitable to disk [{}].", pool, dskCh); if (avoid.shouldAvoid(pool)) { - logger.debug(String.format("StoragePool [%s] is in avoid set, skipping this pool to allocation of disk [%s].", pool, dskCh)); + logger.debug("StoragePool [{}] is in avoid set, skipping this pool to allocation of disk [{}].", pool, dskCh); return false; } if (dskCh.requiresEncryption() && !pool.getPoolType().supportsEncryption()) { if (logger.isDebugEnabled()) { - logger.debug(String.format("Storage pool type '%s' doesn't support encryption required for volume, skipping this pool", pool.getPoolType())); + logger.debug("Storage pool type '[{}]' doesn't support encryption required for volume, skipping this pool", pool.getPoolType()); } return false; } @@ -319,8 +313,8 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement } if (!checkDiskProvisioningSupport(dskCh, pool)) { - logger.debug(String.format("Storage pool [%s] does not have support to disk provisioning of disk [%s].", pool, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(dskCh, - "type", "name", "diskOfferingId", "templateId", "volumeId", "provisioningType", "hyperType"))); + logger.debug("Storage pool [{}] does not have support to disk provisioning of disk [{}].", pool, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(dskCh, + "type", "name", "diskOfferingId", "templateId", "volumeId", "provisioningType", "hyperType")); return false; } @@ -332,7 +326,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement HostVO plannedHost = hostDao.findById(plan.getHostId()); if (!storageMgr.checkIfHostAndStoragePoolHasCommonStorageAccessGroups(plannedHost, pool)) { if (logger.isDebugEnabled()) { - logger.debug(String.format("StoragePool %s and host %s does not have matching storage access groups", pool, plannedHost)); + logger.debug("StoragePool [{}] and host [{}] does not have matching storage access groups", pool, plannedHost); } return false; } @@ -343,13 +337,13 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement if (!isTempVolume) { volume = volumeDao.findById(dskCh.getVolumeId()); if (!storageMgr.storagePoolCompatibleWithVolumePool(pool, volume)) { - logger.debug(String.format("Pool [%s] is not compatible with volume [%s], skipping it.", pool, volume)); + logger.debug("Pool [{}] is not compatible with volume [{}], skipping it.", pool, volume); return false; } } if (pool.isManaged() && !storageUtil.managedStoragePoolCanScale(pool, plan.getClusterId(), plan.getHostId())) { - logger.debug(String.format("Cannot allocate pool [%s] to volume [%s] because the max number of managed clustered filesystems has been exceeded.", pool, volume)); + logger.debug("Cannot allocate pool [{}] to volume [{}] because the max number of managed clustered filesystems has been exceeded.", pool, volume); return false; } @@ -358,13 +352,13 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement requestVolumeDiskProfilePairs.add(new Pair<>(volume, dskCh)); if (dskCh.getHypervisorType() == HypervisorType.VMware) { if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster && storageMgr.isStoragePoolDatastoreClusterParent(pool)) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is a parent datastore cluster.", pool, volume)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is a parent datastore cluster.", pool, volume); return false; } if (pool.getParent() != 0L) { StoragePoolVO datastoreCluster = storagePoolDao.findById(pool.getParent()); if (datastoreCluster == null || (datastoreCluster != null && datastoreCluster.getStatus() != StoragePoolStatus.Up)) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not in [%s] state.", datastoreCluster, volume, StoragePoolStatus.Up)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is not in [{}] state.", datastoreCluster, volume, StoragePoolStatus.Up); return false; } } @@ -374,11 +368,11 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement storageMgr.isStoragePoolCompliantWithStoragePolicy(dskCh.getDiskOfferingId(), pool) : storageMgr.isStoragePoolCompliantWithStoragePolicy(requestVolumeDiskProfilePairs, pool); if (!isStoragePoolStoragePolicyCompliance) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not compliant with the storage policy required by the volume.", pool, volume)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is not compliant with the storage policy required by the volume.", pool, volume); return false; } } catch (StorageUnavailableException e) { - logger.warn(String.format("Could not verify storage policy compliance against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + logger.warn("Could not verify storage policy compliance against storage pool [{}] due to exception [{}]", pool.getUuid(), e.getMessage()); return false; } } @@ -427,19 +421,19 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement protected void logDisabledStoragePools(long dcId, Long podId, Long clusterId, ScopeType scope) { List disabledPools = storagePoolDao.findDisabledPoolsByScope(dcId, podId, clusterId, scope); if (disabledPools != null && !disabledPools.isEmpty()) { - logger.trace(String.format("Ignoring pools [%s] as they are in disabled state.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(disabledPools))); + logger.trace("Ignoring pools [{}] as they are in disabled state.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(disabledPools)); } } protected void logStartOfSearch(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, int returnUpTo, boolean bypassStorageTypeCheck){ - logger.trace(String.format("%s is looking for storage pools that match the VM's disk profile [%s], virtual machine profile [%s] and " - + "deployment plan [%s]. Returning up to [%d] and bypassStorageTypeCheck [%s].", this.getClass().getSimpleName(), dskCh, vmProfile, plan, returnUpTo, bypassStorageTypeCheck)); + logger.trace("[{}] is looking for storage pools that match the VM's disk profile [{}], virtual machine profile [{}] and " + + "deployment plan [{}]. Returning up to [{}] and bypassStorageTypeCheck [{}].", this.getClass().getSimpleName(), dskCh, vmProfile, plan, returnUpTo, bypassStorageTypeCheck); } protected void logEndOfSearch(List storagePoolList) { - logger.debug(String.format("%s is returning [%s] suitable storage pools [%s].", this.getClass().getSimpleName(), storagePoolList.size(), - Arrays.toString(storagePoolList.toArray()))); + logger.debug("[{}] is returning [{}] suitable storage pools [{}].", this.getClass().getSimpleName(), storagePoolList.size(), + Arrays.toString(storagePoolList.toArray())); } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index a2e9eff2a08..26b39e30776 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -230,8 +230,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { updateBuilder.setJobId(answer.getJobId()); updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); updateBuilder.setInstallPath(answer.getInstallPath()); - updateBuilder.setSize(answer.getTemplateSize()); - updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + } _templateStoreDao.update(tmpltStoreVO.getId(), updateBuilder); // update size in vm_template table VMTemplateVO tmlptUpdater = _templateDao.createForUpdate(); @@ -241,8 +243,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { AsyncCompletionCallback caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); @@ -285,19 +286,22 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { updateBuilder.setJobId(answer.getJobId()); updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); updateBuilder.setInstallPath(answer.getInstallPath()); - updateBuilder.setSize(answer.getTemplateSize()); - updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + } _volumeStoreDao.update(volStoreVO.getId(), updateBuilder); // update size in volume table - VolumeVO volUpdater = volumeDao.createForUpdate(); - volUpdater.setSize(answer.getTemplateSize()); - volumeDao.update(obj.getId(), volUpdater); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + VolumeVO volUpdater = volumeDao.createForUpdate(); + volUpdater.setSize(answer.getTemplateSize()); + volumeDao.update(obj.getId(), volUpdater); + } } AsyncCompletionCallback caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); diff --git a/framework/db/src/main/java/com/cloud/utils/db/SequenceFetcher.java b/framework/db/src/main/java/com/cloud/utils/db/SequenceFetcher.java index f902fda3bf1..a59b73e5cee 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SequenceFetcher.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SequenceFetcher.java @@ -64,7 +64,7 @@ public class SequenceFetcher { try { return future.get(); } catch (Exception e) { - logger.warn("Unable to get sequeunce for " + tg.table() + ":" + tg.pkColumnValue(), e); + logger.warn("Unable to get sequence for " + tg.table() + ":" + tg.pkColumnValue(), e); return null; } } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java index 5ab54149645..9d76a7e6ec2 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java @@ -83,6 +83,12 @@ public class CreateExtensionCmd extends BaseCmd { description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].endpoint.url=urlvalue") protected Map details; + @Parameter(name = ApiConstants.RESERVED_RESOURCE_DETAILS, type = CommandType.STRING, + description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -115,6 +121,10 @@ public class CreateExtensionCmd extends BaseCmd { return convertDetailsToMap(details); } + public String getReservedResourceDetails() { + return reservedResourceDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java index ded07d2dd32..5baaea1709d 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java @@ -78,6 +78,12 @@ public class UpdateExtensionCmd extends BaseCmd { "if false or not set, no action)") private Boolean cleanupDetails; + @Parameter(name = ApiConstants.RESERVED_RESOURCE_DETAILS, type = CommandType.STRING, + description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -106,6 +112,10 @@ public class UpdateExtensionCmd extends BaseCmd { return cleanupDetails; } + public String getReservedResourceDetails() { + return reservedResourceDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index 4171b9615fe..1422338ddc9 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -216,6 +216,11 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject AccountService accountService; + // Map of in-built extension names and their reserved resource details that shouldn't be accessible to end-users + protected static final Map> INBUILT_RESERVED_RESOURCE_DETAILS = Map.of( + "proxmox", List.of("proxmox_vmid") + ); + private ScheduledExecutorService extensionPathStateCheckExecutor; protected String getDefaultExtensionRelativePath(String name) { @@ -563,6 +568,25 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana updateExtensionPathReady(extension, true); } + protected void addInbuiltExtensionReservedResourceDetails(long extensionId, List reservedResourceDetails) { + ExtensionVO vo = extensionDao.findById(extensionId); + if (vo == null || vo.isUserDefined()) { + return; + } + String lowerName = StringUtils.defaultString(vo.getName()).toLowerCase(); + Optional>> match = INBUILT_RESERVED_RESOURCE_DETAILS.entrySet().stream() + .filter(e -> lowerName.contains(e.getKey().toLowerCase())) + .findFirst(); + if (match.isPresent()) { + Set existing = new HashSet<>(reservedResourceDetails); + for (String detailKey : match.get().getValue()) { + if (existing.add(detailKey)) { + reservedResourceDetails.add(detailKey); + } + } + } + } + @Override public String getExtensionsPath() { return externalProvisioner.getExtensionsPath(); @@ -577,6 +601,7 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana String relativePath = cmd.getPath(); final Boolean orchestratorRequiresPrepareVm = cmd.isOrchestratorRequiresPrepareVm(); final String stateStr = cmd.getState(); + final String reservedResourceDetails = cmd.getReservedResourceDetails(); ExtensionVO extensionByName = extensionDao.findByName(name); if (extensionByName != null) { throw new CloudRuntimeException("Extension by name already exists"); @@ -624,6 +649,10 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, String.valueOf(orchestratorRequiresPrepareVm), false)); } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + detailsVOList.add(new ExtensionDetailsVO(extension.getId(), + ApiConstants.RESERVED_RESOURCE_DETAILS, reservedResourceDetails, false)); + } if (CollectionUtils.isNotEmpty(detailsVOList)) { extensionDetailsDao.saveDetails(detailsVOList); } @@ -704,6 +733,7 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana final String stateStr = cmd.getState(); final Map details = cmd.getDetails(); final Boolean cleanupDetails = cmd.isCleanupDetails(); + final String reservedResourceDetails = cmd.getReservedResourceDetails(); final ExtensionVO extensionVO = extensionDao.findById(id); if (extensionVO == null) { throw new InvalidParameterValueException("Failed to find the extension"); @@ -732,7 +762,8 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana throw new CloudRuntimeException(String.format("Failed to updated the extension: %s", extensionVO.getName())); } - updateExtensionsDetails(cleanupDetails, details, orchestratorRequiresPrepareVm, id); + updateExtensionsDetails(cleanupDetails, details, orchestratorRequiresPrepareVm, reservedResourceDetails, + id); return extensionVO; }); if (StringUtils.isNotBlank(stateStr)) { @@ -748,9 +779,11 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana return result; } - protected void updateExtensionsDetails(Boolean cleanupDetails, Map details, Boolean orchestratorRequiresPrepareVm, long id) { + protected void updateExtensionsDetails(Boolean cleanupDetails, Map details, + Boolean orchestratorRequiresPrepareVm, String reservedResourceDetails, long id) { final boolean needToUpdateAllDetails = Boolean.TRUE.equals(cleanupDetails) || MapUtils.isNotEmpty(details); - if (!needToUpdateAllDetails && orchestratorRequiresPrepareVm == null) { + if (!needToUpdateAllDetails && orchestratorRequiresPrepareVm == null && + StringUtils.isBlank(reservedResourceDetails)) { return; } if (needToUpdateAllDetails) { @@ -761,6 +794,9 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana hiddenDetails.put(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, String.valueOf(orchestratorRequiresPrepareVm)); } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + hiddenDetails.put(ApiConstants.RESERVED_RESOURCE_DETAILS, reservedResourceDetails); + } if (MapUtils.isNotEmpty(hiddenDetails)) { hiddenDetails.forEach((key, value) -> detailsVOList.add( new ExtensionDetailsVO(id, key, value, false))); @@ -775,15 +811,29 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana extensionDetailsDao.removeDetails(id); } } else { - ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(id, - ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM); - if (detailsVO == null) { - extensionDetailsDao.persist(new ExtensionDetailsVO(id, - ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, - String.valueOf(orchestratorRequiresPrepareVm), false)); - } else if (Boolean.parseBoolean(detailsVO.getValue()) != orchestratorRequiresPrepareVm) { - detailsVO.setValue(String.valueOf(orchestratorRequiresPrepareVm)); - extensionDetailsDao.update(detailsVO.getId(), detailsVO); + if (orchestratorRequiresPrepareVm != null) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(id, + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM); + if (detailsVO == null) { + extensionDetailsDao.persist(new ExtensionDetailsVO(id, + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, + String.valueOf(orchestratorRequiresPrepareVm), false)); + } else if (Boolean.parseBoolean(detailsVO.getValue()) != orchestratorRequiresPrepareVm) { + detailsVO.setValue(String.valueOf(orchestratorRequiresPrepareVm)); + extensionDetailsDao.update(detailsVO.getId(), detailsVO); + } + } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(id, + ApiConstants.RESERVED_RESOURCE_DETAILS); + if (detailsVO == null) { + extensionDetailsDao.persist(new ExtensionDetailsVO(id, + ApiConstants.RESERVED_RESOURCE_DETAILS, + reservedResourceDetails, false)); + } else if (!reservedResourceDetails.equals(detailsVO.getValue())) { + detailsVO.setValue(reservedResourceDetails); + extensionDetailsDao.update(detailsVO.getId(), detailsVO); + } } } } @@ -961,12 +1011,16 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana hiddenDetails = extensionDetails.second(); } else { hiddenDetails = extensionDetailsDao.listDetailsKeyPairs(extension.getId(), - List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM)); + List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, + ApiConstants.RESERVED_RESOURCE_DETAILS)); } if (hiddenDetails.containsKey(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM)) { response.setOrchestratorRequiresPrepareVm(Boolean.parseBoolean( hiddenDetails.get(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM))); } + if (hiddenDetails.containsKey(ApiConstants.RESERVED_RESOURCE_DETAILS)) { + response.setReservedResourceDetails(hiddenDetails.get(ApiConstants.RESERVED_RESOURCE_DETAILS)); + } response.setObjectName(Extension.class.getSimpleName().toLowerCase()); return response; } @@ -1605,6 +1659,24 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana return extensionDao.findById(extensionId); } + @Override + public List getExtensionReservedResourceDetails(long extensionId) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(extensionId, + ApiConstants.RESERVED_RESOURCE_DETAILS); + if (detailsVO == null || !StringUtils.isNotBlank(detailsVO.getValue())) { + return Collections.emptyList(); + } + List reservedDetails = new ArrayList<>(); + String[] parts = detailsVO.getValue().split(","); + for (String part : parts) { + if (StringUtils.isNotBlank(part)) { + reservedDetails.add(part.trim()); + } + } + addInbuiltExtensionReservedResourceDetails(extensionId, reservedDetails); + return reservedDetails; + } + @Override public boolean start() { long pathStateCheckInterval = PathStateCheckInterval.value(); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java index 2edb6ea48e3..2f630966056 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java @@ -94,4 +94,18 @@ public class CreateExtensionCmdTest { setField(cmd, "details", details); assertTrue(MapUtils.isNotEmpty(cmd.getDetails())); } + + @Test + public void getReservedResourceDetailsReturnsValueWhenSet() { + setField(cmd, "reservedResourceDetails", "detail1,detail2,detail3"); + String result = cmd.getReservedResourceDetails(); + assertEquals("detail1,detail2,detail3", result); + } + + @Test + public void getReservedResourceDetailsReturnsNullWhenNotSet() { + setField(cmd, "reservedResourceDetails", null); + String result = cmd.getReservedResourceDetails(); + assertNull(result); + } } diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java index f0a3a6fcf21..5c5c2014a52 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.springframework.test.util.ReflectionTestUtils.setField; import java.util.EnumSet; import java.util.HashMap; @@ -134,6 +135,20 @@ public class UpdateExtensionCmdTest { assertTrue(cmd.isCleanupDetails()); } + @Test + public void getReservedResourceDetailsReturnsValueWhenSet() { + setField(cmd, "reservedResourceDetails", "detail1,detail2,detail3"); + String result = cmd.getReservedResourceDetails(); + assertEquals("detail1,detail2,detail3", result); + } + + @Test + public void getReservedResourceDetailsReturnsNullWhenNotSet() { + setField(cmd, "reservedResourceDetails", null); + String result = cmd.getReservedResourceDetails(); + assertNull(result); + } + @Test public void executeSetsExtensionResponseWhenManagerSucceeds() { Extension extension = mock(Extension.class); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java index 085ae212b28..ff3fce06b00 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java @@ -23,11 +23,13 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -40,6 +42,7 @@ import static org.mockito.Mockito.when; import java.io.File; import java.security.InvalidParameterException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -49,8 +52,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.user.AccountService; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; @@ -85,9 +86,11 @@ import org.apache.cloudstack.framework.extensions.dao.ExtensionResourceMapDao; import org.apache.cloudstack.framework.extensions.dao.ExtensionResourceMapDetailsDao; import org.apache.cloudstack.framework.extensions.vo.ExtensionCustomActionDetailsVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionCustomActionVO; +import org.apache.cloudstack.framework.extensions.vo.ExtensionDetailsVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionResourceMapVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.collections.CollectionUtils; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -113,6 +116,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; @@ -122,6 +126,7 @@ import com.cloud.org.Cluster; import com.cloud.serializer.GsonHelper; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; +import com.cloud.user.AccountService; import com.cloud.utils.Pair; import com.cloud.utils.UuidUtils; import com.cloud.utils.db.EntityManager; @@ -664,6 +669,8 @@ public class ExtensionsManagerImplTest { when(cmd.getPath()).thenReturn(null); when(cmd.isOrchestratorRequiresPrepareVm()).thenReturn(null); when(cmd.getState()).thenReturn(null); + String reservedResourceDetails = "abc,xyz"; + when(cmd.getReservedResourceDetails()).thenReturn(reservedResourceDetails); when(extensionDao.findByName("ext1")).thenReturn(null); when(extensionDao.persist(any())).thenAnswer(inv -> { ExtensionVO extensionVO = inv.getArgument(0); @@ -671,11 +678,20 @@ public class ExtensionsManagerImplTest { return extensionVO; }); when(managementServerHostDao.listBy(any())).thenReturn(Collections.emptyList()); - + List detailsList = new ArrayList<>(); + doAnswer(inv -> { + List detailsVO = inv.getArgument(0); + detailsList.addAll(detailsVO); + return null; + }).when(extensionDetailsDao).saveDetails(anyList()); Extension ext = extensionsManager.createExtension(cmd); assertEquals("ext1", ext.getName()); verify(extensionDao).persist(any()); + assertTrue(CollectionUtils.isNotEmpty(detailsList)); + assertTrue(detailsList.stream() + .anyMatch(detail -> ApiConstants.RESERVED_RESOURCE_DETAILS.equals(detail.getName()) + && reservedResourceDetails.equals(detail.getValue()))); } @Test @@ -938,14 +954,32 @@ public class ExtensionsManagerImplTest { public void updateExtensionsDetails_SavesDetails_WhenDetailsProvided() { long extensionId = 10L; Map details = Map.of("foo", "bar", "baz", "qux"); - extensionsManager.updateExtensionsDetails(false, details, null, extensionId); + extensionsManager.updateExtensionsDetails(false, details, null, null, extensionId); verify(extensionDetailsDao).saveDetails(any()); } + @Test + public void updateExtensionsDetails_PersistReservedDetail_WhenProvided() { + long extensionId = 10L; + when(extensionDetailsDao.persist(any())).thenReturn(mock(ExtensionDetailsVO.class)); + extensionsManager.updateExtensionsDetails(false, null, null, "abc,xyz", extensionId); + verify(extensionDetailsDao).persist(any()); + } + + @Test + public void updateExtensionsDetails_UpdateReservedDetail_WhenProvided() { + long extensionId = 10L; + when(extensionDetailsDao.findDetail(anyLong(), eq(ApiConstants.RESERVED_RESOURCE_DETAILS))) + .thenReturn(mock(ExtensionDetailsVO.class)); + when(extensionDetailsDao.update(anyLong(), any())).thenReturn(true); + extensionsManager.updateExtensionsDetails(false, null, null, "abc,xyz", extensionId); + verify(extensionDetailsDao).update(anyLong(), any()); + } + @Test public void updateExtensionsDetails_DoesNothing_WhenDetailsAndCleanupAreNull() { long extensionId = 11L; - extensionsManager.updateExtensionsDetails(null, null, null, extensionId); + extensionsManager.updateExtensionsDetails(null, null, null, null, extensionId); verify(extensionDetailsDao, never()).removeDetails(anyLong()); verify(extensionDetailsDao, never()).saveDetails(any()); } @@ -953,7 +987,7 @@ public class ExtensionsManagerImplTest { @Test public void updateExtensionsDetails_RemovesDetailsOnly_WhenCleanupIsTrue() { long extensionId = 12L; - extensionsManager.updateExtensionsDetails(true, null, null, extensionId); + extensionsManager.updateExtensionsDetails(true, null, null, null, extensionId); verify(extensionDetailsDao).removeDetails(extensionId); verify(extensionDetailsDao, never()).saveDetails(any()); } @@ -961,7 +995,7 @@ public class ExtensionsManagerImplTest { @Test public void updateExtensionsDetails_PersistsOrchestratorFlag_WhenFlagIsNotNull() { long extensionId = 13L; - extensionsManager.updateExtensionsDetails(false, null, true, extensionId); + extensionsManager.updateExtensionsDetails(false, null, true, null, extensionId); verify(extensionDetailsDao).persist(any()); } @@ -970,7 +1004,7 @@ public class ExtensionsManagerImplTest { long extensionId = 14L; Map details = Map.of("foo", "bar"); doThrow(CloudRuntimeException.class).when(extensionDetailsDao).saveDetails(any()); - extensionsManager.updateExtensionsDetails(false, details, null, extensionId); + extensionsManager.updateExtensionsDetails(false, details, null, null, extensionId); } @Test @@ -1161,7 +1195,8 @@ public class ExtensionsManagerImplTest { when(externalProvisioner.getExtensionPath("entry2.sh")).thenReturn("/some/path/entry2.sh"); Map hiddenDetails = Map.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, "false"); - when(extensionDetailsDao.listDetailsKeyPairs(2L, List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM))) + when(extensionDetailsDao.listDetailsKeyPairs(2L, List.of( + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, ApiConstants.RESERVED_RESOURCE_DETAILS))) .thenReturn(hiddenDetails); EnumSet viewDetails = EnumSet.noneOf(ApiConstants.ExtensionDetails.class); @@ -2069,4 +2104,118 @@ public class ExtensionsManagerImplTest { } } + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenDetailsNotFound() { + long extensionId = 1L; + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(null); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenValueIsBlank() { + long extensionId = 2L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(" "); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsListOfTrimmedDetails() { + long extensionId = 3L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(" detail1 , detail2,detail3 "); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("detail1", result.get(0)); + assertEquals("detail2", result.get(1)); + assertEquals("detail3", result.get(2)); + } + + @Test + public void getExtensionReservedResourceDetailsHandlesEmptyPartsGracefully() { + long extensionId = 4L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn("detail1,,detail2, ,detail3"); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("detail1", result.get(0)); + assertEquals("detail2", result.get(1)); + assertEquals("detail3", result.get(2)); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenSplitResultsInNoParts() { + long extensionId = 5L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(","); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingWhenExtensionNotFound() { + when(extensionDao.findById(1L)).thenReturn(null); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(1L, reservedResourceDetails); + assertTrue(reservedResourceDetails.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingForUserDefinedExtension() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(true); + when(extensionDao.findById(2L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + reservedResourceDetails.add("existing-detail"); + extensionsManager.addInbuiltExtensionReservedResourceDetails(2L, reservedResourceDetails); + assertEquals(1, reservedResourceDetails.size()); + assertTrue(reservedResourceDetails.contains("existing-detail")); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingWhenNoMatchFound() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(false); + when(extension.getName()).thenReturn("no-such-inbuilt-key-expected"); + when(extensionDao.findById(3L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(3L, reservedResourceDetails); + assertTrue(reservedResourceDetails.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsAddedDetails() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(false); + Map.Entry> entry = + ExtensionsManagerImpl.INBUILT_RESERVED_RESOURCE_DETAILS.entrySet().iterator().next(); + when(extension.getName()).thenReturn(entry.getKey()); + when(extensionDao.findById(3L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(3L, reservedResourceDetails); + assertFalse(reservedResourceDetails.isEmpty()); + assertEquals(reservedResourceDetails.size(), entry.getValue().size()); + assertTrue(reservedResourceDetails.containsAll(entry.getValue())); + } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 9f7a4ad6e05..926280bfead 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -23,12 +23,30 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import com.cloud.utils.db.GenericDao; +import javax.annotation.Nullable; + public interface AsyncJobDao extends GenericDao { AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId); List findInstancePendingAsyncJobs(String instanceType, Long accountId); + /** + * Finds async job matching the given parameters. + * Non-null parameters are added to search criteria. + * Returns the most recent job by creation date. + *

+ * When searching by resourceId and resourceType, only one active job + * is expected per resource, so returning a single result is sufficient. + * + * @param id job ID + * @param resourceId resource ID (instanceId) + * @param resourceType resource type (instanceType) + * @return matching job or null + */ + @Nullable + AsyncJobVO findJob(Long id, Long resourceId, String resourceType); + AsyncJobVO findPseudoJob(long threadId, long msid); void cleanupPseduoJobs(long msid); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index a2f1f36b863..81cc5d4f2a8 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -22,6 +22,8 @@ import java.util.Date; import java.util.List; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.jobs.JobInfo; @@ -45,6 +47,7 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements private final SearchBuilder expiringUnfinishedAsyncJobSearch; private final SearchBuilder expiringCompletedAsyncJobSearch; private final SearchBuilder failureMsidAsyncJobSearch; + private final SearchBuilder byIdResourceIdResourceTypeSearch; private final GenericSearchBuilder asyncJobTypeSearch; private final GenericSearchBuilder pendingNonPseudoAsyncJobsSearch; @@ -95,6 +98,12 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements failureMsidAsyncJobSearch.and("job_cmd", failureMsidAsyncJobSearch.entity().getCmd(), Op.IN); failureMsidAsyncJobSearch.done(); + byIdResourceIdResourceTypeSearch = createSearchBuilder(); + byIdResourceIdResourceTypeSearch.and("id", byIdResourceIdResourceTypeSearch.entity().getId(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.and("instanceId", byIdResourceIdResourceTypeSearch.entity().getInstanceId(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.and("instanceType", byIdResourceIdResourceTypeSearch.entity().getInstanceType(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.done(); + asyncJobTypeSearch = createSearchBuilder(Long.class); asyncJobTypeSearch.select(null, SearchCriteria.Func.COUNT, asyncJobTypeSearch.entity().getId()); asyncJobTypeSearch.and("job_info", asyncJobTypeSearch.entity().getCmdInfo(),Op.LIKE); @@ -140,6 +149,30 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements return listBy(sc); } + @Override + public AsyncJobVO findJob(Long id, Long resourceId, String resourceType) { + SearchCriteria sc = byIdResourceIdResourceTypeSearch.create(); + + if (id == null && resourceId == null && StringUtils.isBlank(resourceType)) { + logger.debug("findJob called with all null parameters"); + return null; + } + + if (id != null) { + sc.setParameters("id", id); + } + if (resourceId != null && StringUtils.isNotBlank(resourceType)) { + sc.setParameters("instanceType", resourceType); + sc.setParameters("instanceId", resourceId); + } + Filter filter = new Filter(AsyncJobVO.class, "created", false, 0L, 1L); + List result = searchIncludingRemoved(sc, filter, Boolean.FALSE, false); + if (CollectionUtils.isNotEmpty(result)) { + return result.get(0); + } + return null; + } + @Override public AsyncJobVO findPseudoJob(long threadId, long msid) { SearchCriteria sc = pseudoJobSearch.create(); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java new file mode 100644 index 00000000000..cc6ca203ef7 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaAccountStateFilter.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota; + +import org.apache.commons.lang3.StringUtils; + +public enum QuotaAccountStateFilter { + ALL, ACTIVE, REMOVED; + + public static QuotaAccountStateFilter getValue(String value) { + if (StringUtils.isBlank(value)) { + return null; + } + for (QuotaAccountStateFilter state : values()) { + if (state.name().equalsIgnoreCase(value)) { + return state; + } + } + return null; + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java new file mode 100644 index 00000000000..d8ee2607501 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDao.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.dao; + +import java.util.List; + +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.GenericDao; + +public interface QuotaSummaryDao extends GenericDao { + + Pair, Integer> listQuotaSummariesForAccountAndOrDomain(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter, Long startIndex, Long pageSize); +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java new file mode 100644 index 00000000000..d90d5a75859 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaSummaryDaoImpl.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.dao; + +import java.util.List; + +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; + +import com.cloud.utils.Pair; +import com.cloud.utils.db.Filter; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +public class QuotaSummaryDaoImpl extends GenericDaoBase implements QuotaSummaryDao { + + @Override + public Pair, Integer> listQuotaSummariesForAccountAndOrDomain(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter, Long startIndex, Long pageSize) { + SearchCriteria searchCriteria = createListQuotaSummariesSearchCriteria(accountId, accountName, domainId, domainPath, accountStateFilter); + Filter filter = new Filter(QuotaSummaryVO.class, "accountName", true, startIndex, pageSize); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback, Integer>>) status -> searchAndCount(searchCriteria, filter)); + } + + protected SearchCriteria createListQuotaSummariesSearchCriteria(Long accountId, String accountName, Long domainId, String domainPath, + QuotaAccountStateFilter accountStateFilter) { + SearchCriteria searchCriteria = createListQuotaSummariesSearchBuilder(accountStateFilter).create(); + + searchCriteria.setParametersIfNotNull("accountId", accountId); + searchCriteria.setParametersIfNotNull("domainId", domainId); + + if (accountName != null) { + searchCriteria.setParameters("accountName", "%" + accountName + "%"); + } + + if (domainPath != null) { + searchCriteria.setParameters("domainPath", domainPath + "%"); + } + + return searchCriteria; + } + + protected SearchBuilder createListQuotaSummariesSearchBuilder(QuotaAccountStateFilter accountStateFilter) { + SearchBuilder searchBuilder = createSearchBuilder(); + + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("accountName", searchBuilder.entity().getAccountName(), SearchCriteria.Op.LIKE); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainPath", searchBuilder.entity().getDomainPath(), SearchCriteria.Op.LIKE); + + if (QuotaAccountStateFilter.REMOVED.equals(accountStateFilter)) { + searchBuilder.and("accountRemoved", searchBuilder.entity().getAccountRemoved(), SearchCriteria.Op.NNULL); + } else if (QuotaAccountStateFilter.ACTIVE.equals(accountStateFilter)) { + searchBuilder.and("accountRemoved", searchBuilder.entity().getAccountRemoved(), SearchCriteria.Op.NULL); + } + + return searchBuilder; + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java new file mode 100644 index 00000000000..f9796497d57 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaSummaryVO.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; +import java.util.Date; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; + +import com.cloud.user.Account; + +@Entity +@Table(name = "quota_summary_view") +public class QuotaSummaryVO { + + @Id + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "quota_enforce") + private Integer quotaEnforce = 0; + + @Column(name = "quota_balance") + private BigDecimal quotaBalance; + + @Column(name = "quota_balance_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date quotaBalanceDate = null; + + @Column(name = "quota_min_balance") + private BigDecimal quotaMinBalance; + + @Column(name = "quota_alert_type") + private Integer quotaAlertType = null; + + @Column(name = "quota_alert_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date quotaAlertDate = null; + + @Column(name = "last_statement_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date lastStatementDate = null; + + @Column(name = "account_uuid") + private String accountUuid; + + @Column(name = "account_name") + private String accountName; + + @Column(name = "account_state") + @Enumerated(EnumType.STRING) + private Account.State accountState; + + @Column(name = "account_removed") + private Date accountRemoved; + + @Column(name = "domain_id") + private Long domainId; + + @Column(name = "domain_uuid") + private String domainUuid; + + @Column(name = "domain_name") + private String domainName; + + @Column(name = "domain_path") + private String domainPath; + + @Column(name = "domain_removed") + private Date domainRemoved; + + @Column(name = "project_uuid") + private String projectUuid; + + @Column(name = "project_name") + private String projectName; + + @Column(name = "project_removed") + private Date projectRemoved; + + public Long getAccountId() { + return accountId; + } + + public BigDecimal getQuotaBalance() { + return quotaBalance; + } + + public String getAccountUuid() { + return accountUuid; + } + + public String getAccountName() { + return accountName; + } + + public Date getAccountRemoved() { + return accountRemoved; + } + + public Account.State getAccountState() { + return accountState; + } + + public Long getDomainId() { + return domainId; + } + + public String getDomainUuid() { + return domainUuid; + } + + public String getDomainPath() { + return domainPath; + } + + public Date getDomainRemoved() { + return domainRemoved; + } + + public String getProjectUuid() { + return projectUuid; + } + + public String getProjectName() { + return projectName; + } + + public Date getProjectRemoved() { + return projectRemoved; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 453355c8522..304b23b7220 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -19,7 +19,8 @@ - + + 1.8.3 or libgcrypt20) Group: System Environment/Libraries @@ -334,11 +336,11 @@ cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json ln -sf /etc/%{name}/ui/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json -# Package mysql-connector-python -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/e9/93/4860cebd5ad3ff2664ad3c966490ccb46e3b88458b2095145bca11727ca4/setuptools-47.3.1-py3-none-any.whl -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/32/27/1141a8232723dcb10a595cc0ce4321dcbbd5215300bf4acfc142343205bf/protobuf-3.19.6-py2.py3-none-any.whl +# Package mysql-connector-python (bundled to avoid dependency on external community repo) +# Version 8.0.31 is the last version supporting Python 3.6 (EL8) wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Version 8.3.0 supports Python 3.8 to 3.12 (EL9, EL10) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/53/ed/26a4b8cacb8852c6fd97d2d58a7f2591c41989807ea82bd8d9725a4e6937/mysql_connector_python-8.3.0-py2.py3-none-any.whl chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt @@ -455,8 +457,13 @@ then fi %post management -# Install mysql-connector-python -pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.19.6-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Install mysql-connector-python wheel +# Detect Python version to install compatible wheel +if python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 7) else 1)'; then + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.3.0-py2.py3-none-any.whl +else + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +fi /usr/bin/systemctl enable cloudstack-management > /dev/null 2>&1 || true /usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index c5b690cfcd4..df9336026f4 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -357,7 +357,8 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co volumePools.add(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null); String volumePathPrefix = getVolumePathPrefix(storagePool); - volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath())); + String volumePathSuffix = getVolumePathSuffix(storagePool); + volumePaths.add(String.format("%s%s%s", volumePathPrefix, volume.getPath(), volumePathSuffix)); } return new Pair<>(volumePools, volumePaths); } @@ -367,14 +368,24 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co if (ScopeType.HOST.equals(storagePool.getScope()) || Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) || Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) { - volumePathPrefix = storagePool.getPath(); + volumePathPrefix = storagePool.getPath() + "/"; + } else if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + volumePathPrefix = "/dev/drbd/by-res/cs-"; } else { // Should be Storage.StoragePoolType.NetworkFilesystem - volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + volumePathPrefix = String.format("/mnt/%s/", storagePool.getUuid()); } return volumePathPrefix; } + private String getVolumePathSuffix(StoragePoolVO storagePool) { + if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + return "/0"; + } else { + return ""; + } + } + @Override public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); @@ -419,7 +430,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vmNameAndState.first()); - restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID))); + String restoreVolumePath = String.format("%s%s%s", getVolumePathPrefix(pool), volumeUUID, getVolumePathSuffix(pool)); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(restoreVolumePath)); + restoreCommand.setRestoreVolumeSizes(Collections.singletonList(backedUpVolumeSize)); DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null)); restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT)); @@ -478,6 +491,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } + if (host == null) { + throw new CloudRuntimeException(String.format("Unable to find a running KVM host in zone %d to delete backup %s", backup.getZoneId(), backup.getUuid())); + } DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions()); @@ -564,7 +580,14 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public void syncBackupStorageStats(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + if (CollectionUtils.isEmpty(repositories)) { + return; + } final Host host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + if (host == null) { + logger.warn("Unable to find a running KVM host in zone {} to sync backup storage stats", zoneId); + return; + } for (final BackupRepository repository : repositories) { GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions()); BackupStorageStatsAnswer answer; diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java index 87322b01f4d..870b9b6df6e 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaSummaryCmd.java @@ -16,61 +16,80 @@ //under the License. package org.apache.cloudstack.api.command; -import com.cloud.user.Account; import com.cloud.utils.Pair; + +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; -import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaSummaryResponse; -import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.api.response.ProjectResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.quota.QuotaAccountStateFilter; +import org.apache.cloudstack.quota.QuotaService; +import org.apache.commons.lang3.ObjectUtils; import java.util.List; import javax.inject.Inject; -@APICommand(name = "quotaSummary", responseObject = QuotaSummaryResponse.class, description = "Lists balance and quota usage for all Accounts", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaSummary", responseObject = QuotaSummaryResponse.class, description = "Lists Quota balance summary of Accounts and Projects.", since = "4.7.0", + requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaSummaryCmd extends BaseListCmd { - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = false, description = "Optional, Account Id for which statement needs to be generated") - private String accountName; - - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = false, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") - private Long domainId; - - @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, required = false, description = "Optional, to list all Accounts irrespective of the quota activity") - private Boolean listAll; + @Inject + QuotaResponseBuilder quotaResponseBuilder; @Inject - QuotaResponseBuilder _responseBuilder; + QuotaService quotaService; - public QuotaSummaryCmd() { - super(); - } + @ACL + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which balance will be listed. Can not be specified with projectid.", since = "4.23.0") + private Long accountId; + + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = false, description = "Name of the Account for which balance will be listed.") + private String accountName; + + @ACL + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = false, entityType = DomainResponse.class, description = "ID of the Domain for which balance will be listed. May be used individually or with accountname.") + private Long domainId; + + @Parameter(name = ApiConstants.LIST_ALL, type = CommandType.BOOLEAN, description = "False (default) lists the Quota balance summary for calling Account. True lists balance summary for " + + "Accounts which the caller has access. If domain ID is informed, this parameter is considered as true.") + private Boolean listAll; + + @Parameter(name = ApiConstants.ACCOUNT_STATE_TO_SHOW, type = CommandType.STRING, description = "Possible values are [ALL, ACTIVE, REMOVED]. ALL will list summaries for " + + "active and removed accounts; ACTIVE will list summaries only for active accounts; REMOVED will list summaries only for removed accounts. The default value is ACTIVE.", + since = "4.23.0") + private String accountStateToShow; + + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which balance will be listed. Can not be specified with accountId.", since = "4.23.0") + private Long projectId; @Override public void execute() { - Account caller = CallContext.current().getCallingAccount(); - Pair, Integer> responses; - if (caller.getType() == Account.Type.ADMIN) { - if (getAccountName() != null && getDomainId() != null) - responses = _responseBuilder.createQuotaSummaryResponse(getAccountName(), getDomainId()); - else - responses = _responseBuilder.createQuotaSummaryResponse(isListAll(), getKeyword(), getStartIndex(), getPageSizeVal()); - } else { - responses = _responseBuilder.createQuotaSummaryResponse(caller.getAccountName(), caller.getDomainId()); - } - final ListResponse response = new ListResponse(); + Pair, Integer> responses = quotaResponseBuilder.createQuotaSummaryResponse(this); + ListResponse response = new ListResponse<>(); response.setResponses(responses.first(), responses.second()); response.setResponseName(getCommandName()); setResponseObject(response); } + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + public String getAccountName() { return accountName; } @@ -88,16 +107,31 @@ public class QuotaSummaryCmd extends BaseListCmd { } public Boolean isListAll() { - return listAll == null ? false: listAll; + // If a domain ID was specified, then allow listing all summaries of domain + return ObjectUtils.defaultIfNull(listAll, Boolean.FALSE) || domainId != null; } public void setListAll(Boolean listAll) { this.listAll = listAll; } - @Override - public long getEntityOwnerId() { - return Account.ACCOUNT_ID_SYSTEM; + public Long getProjectId() { + return projectId; } + public QuotaAccountStateFilter getAccountStateToShow() { + QuotaAccountStateFilter state = QuotaAccountStateFilter.getValue(accountStateToShow); + if (state != null) { + return state; + } + return QuotaAccountStateFilter.ACTIVE; + } + + @Override + public long getEntityOwnerId() { + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; + } + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 67f75ebf82f..177fb00d4b5 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -24,6 +24,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; @@ -52,11 +53,7 @@ public interface QuotaResponseBuilder { QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); - Pair, Integer> createQuotaSummaryResponse(Boolean listAll); - - Pair, Integer> createQuotaSummaryResponse(Boolean listAll, String keyword, Long startIndex, Long pageSize); - - Pair, Integer> createQuotaSummaryResponse(String accountName, Long domainId); + Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index f9456587058..2d6ec3255f4 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -42,11 +42,14 @@ import java.util.stream.Collectors; import javax.inject.Inject; +import com.cloud.domain.Domain; import com.cloud.exception.PermissionDeniedException; +import com.cloud.projects.dao.ProjectDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaBalanceCmd; @@ -56,6 +59,7 @@ import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; import org.apache.cloudstack.api.command.QuotaPresetVariablesListCmd; import org.apache.cloudstack.api.command.QuotaStatementCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; @@ -74,11 +78,13 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariable import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; + import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaCreditsDao; import org.apache.cloudstack.quota.dao.QuotaEmailConfigurationDao; import org.apache.cloudstack.quota.dao.QuotaEmailTemplatesDao; +import org.apache.cloudstack.quota.dao.QuotaSummaryDao; import org.apache.cloudstack.quota.dao.QuotaTariffDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; @@ -86,10 +92,13 @@ import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; +import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.compress.utils.Sets; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.reflect.FieldUtils; @@ -108,7 +117,6 @@ import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.user.dao.UserDao; import com.cloud.utils.Pair; -import com.cloud.utils.db.Filter; @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @@ -121,7 +129,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private QuotaCreditsDao quotaCreditsDao; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageDao quotaUsageDao; @Inject private QuotaEmailTemplatesDao _quotaEmailTemplateDao; @@ -132,29 +140,29 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject + private ProjectDao projectDao; + @Inject private QuotaAccountDao quotaAccountDao; @Inject - private DomainDao _domainDao; + private DomainDao domainDao; @Inject private AccountManager _accountMgr; @Inject - private QuotaStatement _statement; + private QuotaStatement quotaStatement; @Inject private QuotaManager _quotaManager; @Inject private QuotaEmailConfigurationDao quotaEmailConfigurationDao; @Inject + private QuotaSummaryDao quotaSummaryDao; + @Inject private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; - protected void checkActivationRulesAllowed(String activationRule) { - if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) { - throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration."); - } - } + private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); @Override public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) { @@ -180,75 +188,113 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public Pair, Integer> createQuotaSummaryResponse(final String accountName, final Long domainId) { - List result = new ArrayList(); + public Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd) { + Account caller = CallContext.current().getCallingAccount(); - if (accountName != null && domainId != null) { - Account account = _accountDao.findActiveAccount(accountName, domainId); - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); + if (!accountTypesThatCanListAllQuotaSummaries.contains(caller.getType()) || !cmd.isListAll()) { + return getQuotaSummaryResponse(cmd.getEntityOwnerId(), null, null, cmd); } - return new Pair<>(result, result.size()); + return getQuotaSummaryResponseWithListAll(cmd, caller); } - @Override - public Pair, Integer> createQuotaSummaryResponse(Boolean listAll) { - return createQuotaSummaryResponse(listAll, null, null, null); - } - - @Override - public Pair, Integer> createQuotaSummaryResponse(Boolean listAll, final String keyword, final Long startIndex, final Long pageSize) { - List result = new ArrayList(); - Integer count = 0; - if (listAll) { - Filter filter = new Filter(AccountVO.class, "accountName", true, startIndex, pageSize); - Pair, Integer> data = _accountDao.findAccountsLike(keyword, filter); - count = data.second(); - for (final AccountVO account : data.first()) { - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); - } - } else { - Pair, Integer> data = quotaAccountDao.listAllQuotaAccount(startIndex, pageSize); - count = data.second(); - for (final QuotaAccountVO quotaAccount : data.first()) { - AccountVO account = _accountDao.findById(quotaAccount.getId()); - if (account == null) { - continue; - } - QuotaSummaryResponse qr = getQuotaSummaryResponse(account); - result.add(qr); + protected Pair, Integer> getQuotaSummaryResponseWithListAll(QuotaSummaryCmd cmd, Account caller) { + Long domainId = cmd.getDomainId(); + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + if (domain == null) { + throw new InvalidParameterValueException(String.format("Domain [%s] does not exist.", domainId)); } } - return new Pair<>(result, count); + + String domainPath = getDomainPathByDomainIdForDomainAdmin(caller); + + Long accountId = cmd.getEntityOwnerId(); + if (accountId == -1) { + accountId = cmd.isListAll() ? null : caller.getAccountId(); + } + + return getQuotaSummaryResponse(accountId, domainId, domainPath, cmd); } - protected QuotaSummaryResponse getQuotaSummaryResponse(final Account account) { - Calendar[] period = _statement.getCurrentStatementTime(); - - if (account != null) { - QuotaSummaryResponse qr = new QuotaSummaryResponse(); - DomainVO domain = _domainDao.findById(account.getDomainId()); - BigDecimal curBalance = _quotaBalanceDao.lastQuotaBalance(account.getAccountId(), account.getDomainId(), period[1].getTime()); - BigDecimal quotaUsage = _quotaUsageDao.findTotalQuotaUsage(account.getAccountId(), account.getDomainId(), null, period[0].getTime(), period[1].getTime()); - - qr.setAccountId(account.getUuid()); - qr.setAccountName(account.getAccountName()); - qr.setDomainId(domain.getUuid()); - qr.setDomainName(domain.getName()); - qr.setBalance(curBalance); - qr.setQuotaUsage(quotaUsage); - qr.setState(account.getState()); - qr.setStartDate(period[0].getTime()); - qr.setEndDate(period[1].getTime()); - qr.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - qr.setQuotaEnabled(QuotaConfig.QuotaAccountEnabled.valueIn(account.getId())); - qr.setObjectName("summary"); - return qr; - } else { - return new QuotaSummaryResponse(); + /** + * Retrieves the domain path of the caller's domain (if the caller is Domain Admin) for filtering in the quota summary query. + * @return null if the caller is an Admin or the domain path of the caller's domain if the caller is a Domain Admin. + * @throws InvalidParameterValueException if it cannot find the domain. + */ + protected String getDomainPathByDomainIdForDomainAdmin(Account caller) { + if (caller.getType() != Account.Type.DOMAIN_ADMIN) { + return null; } + + Long domainId = caller.getDomainId(); + Domain domain = domainDao.findById(domainId); + _accountMgr.checkAccess(caller, domain); + + if (domain == null) { + throw new InvalidParameterValueException(String.format("Domain ID [%s] is invalid.", domainId)); + } + + return domain.getPath(); + } + + /** + * Returns a List of QuotaSummaryResponse based on the provided parameters. + * @param accountId ID of the Account to return the summaries for. If -1, either because no specific + * Account was provided, or list all is disabled, then the summary is generated for the calling Account. + * @param domainId ID of the Domain to return the summaries for. + * @param domainPath path of the Domain to return the summaries for. + */ + protected Pair, Integer> getQuotaSummaryResponse(Long accountId, Long domainId, String domainPath, QuotaSummaryCmd cmd) { + if (accountId != null && accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); + } + + Pair, Integer> pairSummaries = quotaSummaryDao.listQuotaSummariesForAccountAndOrDomain(accountId, cmd.getKeyword(), domainId, domainPath, + cmd.getAccountStateToShow(), cmd.getStartIndex(), cmd.getPageSizeVal()); + List summaries = pairSummaries.first(); + + if (CollectionUtils.isEmpty(summaries)) { + logger.info("There are no summaries to list for parameters [{}].", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "domainId", "listAll", "page", "pageSize")); + return new Pair<>(new ArrayList<>(), 0); + } + + List responses = summaries.stream().map(this::getQuotaSummaryResponse).collect(Collectors.toList()); + + return new Pair<>(responses, pairSummaries.second()); + } + + protected QuotaSummaryResponse getQuotaSummaryResponse(QuotaSummaryVO summary) { + QuotaSummaryResponse response = new QuotaSummaryResponse(); + Account account = _accountDao.findByUuidIncludingRemoved(summary.getAccountUuid()); + + Calendar[] period = quotaStatement.getCurrentStatementTime(); + Date startDate = period[0].getTime(); + Date endDate = period[1].getTime(); + BigDecimal quotaUsage = quotaUsageDao.findTotalQuotaUsage(account.getAccountId(), account.getDomainId(), null, startDate, endDate); + + response.setQuotaUsage(quotaUsage); + response.setStartDate(startDate); + response.setEndDate(endDate); + response.setAccountId(summary.getAccountUuid()); + response.setAccountName(summary.getAccountName()); + response.setDomainId(summary.getDomainUuid()); + response.setDomainPath(summary.getDomainPath()); + response.setBalance(summary.getQuotaBalance()); + response.setState(summary.getAccountState()); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setQuotaEnabled(QuotaConfig.QuotaAccountEnabled.valueIn(account.getId())); + response.setDomainRemoved(summary.getDomainRemoved() != null); + response.setAccountRemoved(summary.getAccountRemoved() != null); + response.setObjectName("summary"); + + if (summary.getProjectUuid() != null) { + response.setProjectId(summary.getProjectUuid()); + response.setProjectName(summary.getProjectName()); + response.setProjectRemoved(summary.getProjectRemoved() != null); + } + + return response; } public boolean isUserAllowedToSeeActivationRules(User user) { @@ -450,6 +496,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { Integer position = cmd.getPosition(); warnQuotaTariffUpdateDeprecatedFields(cmd); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); @@ -457,8 +504,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { throw new InvalidParameterValueException(String.format("There is no quota tariffs with name [%s].", name)); } - checkActivationRulesAllowed(activationRule); - Date currentQuotaTariffStartDate = currentQuotaTariff.getEffectiveOn(); currentQuotaTariff.setRemoved(now); @@ -707,14 +752,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { String activationRule = cmd.getActivationRule(); Integer position = ObjectUtils.defaultIfNull(cmd.getPosition(), 1); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); + QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); if (currentQuotaTariff != null) { throw new InvalidParameterValueException(String.format("A quota tariff with name [%s] already exist.", name)); } - checkActivationRulesAllowed(activationRule); - if (startDate.compareTo(now) < 0) { throw new InvalidParameterValueException(String.format("The value passed as Quota tariff's start date is in the past: [%s]. " + "Please, inform a date in the future or do not pass the parameter to use the current date and time.", startDate)); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java index f8f27b4813d..d76202bfd88 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaSummaryResponse.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import com.google.gson.annotations.SerializedName; @@ -30,40 +29,48 @@ import com.cloud.user.Account.State; public class QuotaSummaryResponse extends BaseResponse { @SerializedName("accountid") - @Param(description = "Account ID") + @Param(description = "Account's ID") private String accountId; @SerializedName("account") - @Param(description = "Account name") + @Param(description = "Account's name") private String accountName; @SerializedName("domainid") - @Param(description = "Domain ID") + @Param(description = "Domain's ID") private String domainId; @SerializedName("domain") - @Param(description = "Domain name") - private String domainName; + @Param(description = "Domain's path") + private String domainPath; @SerializedName("balance") - @Param(description = "Account balance") + @Param(description = "Account's balance") private BigDecimal balance; @SerializedName("state") - @Param(description = "Account state") + @Param(description = "Account's state") private State state; + @SerializedName("domainremoved") + @Param(description = "If the domain is removed or not", since = "4.23.0") + private boolean domainRemoved; + + @SerializedName("accountremoved") + @Param(description = "If the account is removed or not", since = "4.23.0") + private boolean accountRemoved; + @SerializedName("quota") - @Param(description = "Quota usage of this period") + @Param(description = "Quota consumed between the startdate and enddate") private BigDecimal quotaUsage; @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; + @Param(description = "Start date of the quota consumption") + private Date startDate; @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; + @Param(description = "End date of the quota consumption") + private Date endDate; @SerializedName("currency") @Param(description = "Currency") @@ -73,9 +80,17 @@ public class QuotaSummaryResponse extends BaseResponse { @Param(description = "If the account has the quota config enabled") private boolean quotaEnabled; - public QuotaSummaryResponse() { - super(); - } + @SerializedName("projectname") + @Param(description = "Name of the project", since = "4.23.0") + private String projectName; + + @SerializedName("projectid") + @Param(description = "Project's id", since = "4.23.0") + private String projectId; + + @SerializedName("projectremoved") + @Param(description = "Whether the project is removed or not", since = "4.23.0") + private Boolean projectRemoved; public String getAccountId() { return accountId; @@ -101,28 +116,16 @@ public class QuotaSummaryResponse extends BaseResponse { this.domainId = domainId; } - public String getDomainName() { - return domainName; - } - - public void setDomainName(String domainName) { - this.domainName = domainName; - } - - public BigDecimal getQuotaUsage() { - return quotaUsage; - } - - public State getState() { - return state; + public void setDomainPath(String domainPath) { + this.domainPath = domainPath; } public void setState(State state) { this.state = state; } - public void setQuotaUsage(BigDecimal startQuota) { - this.quotaUsage = startQuota.setScale(2, RoundingMode.HALF_EVEN); + public void setQuotaUsage(BigDecimal quotaUsage) { + this.quotaUsage = quotaUsage; } public BigDecimal getBalance() { @@ -130,38 +133,42 @@ public class QuotaSummaryResponse extends BaseResponse { } public void setBalance(BigDecimal balance) { - this.balance = balance.setScale(2, RoundingMode.HALF_EVEN); - } - - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + this.balance = balance; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); - } - - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + this.startDate = startDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - public String getCurrency() { - return currency; + this.endDate = endDate; } public void setCurrency(String currency) { this.currency = currency; } - public boolean getQuotaEnabled() { - return quotaEnabled; - } - public void setQuotaEnabled(boolean quotaEnabled) { this.quotaEnabled = quotaEnabled; } + + public void setProjectName(String projectName) { + this.projectName = projectName; + } + + public void setProjectId(String projectId) { + this.projectId = projectId; + } + + public void setProjectRemoved(Boolean projectRemoved) { + this.projectRemoved = projectRemoved; + } + + public void setDomainRemoved(boolean domainRemoved) { + this.domainRemoved = domainRemoved; + } + + public void setAccountRemoved(boolean accountRemoved) { + this.accountRemoved = accountRemoved; + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index f6a34e01be8..a421d0f066a 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -40,6 +40,4 @@ public interface QuotaService extends PluggableService { boolean saveQuotaAccount(AccountVO account, BigDecimal aggrUsage, Date endDate); - boolean isJsInterpretationEnabled(); - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index f455c3cba14..2d7d623d1d9 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -26,6 +26,8 @@ import java.util.TimeZone; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.projects.ProjectManager; +import com.cloud.user.AccountService; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -62,7 +64,6 @@ import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.server.ManagementService; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; @@ -75,6 +76,8 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private AccountDao _accountDao; @Inject + private AccountService accountService; + @Inject private QuotaAccountDao _quotaAcc; @Inject private QuotaUsageDao _quotaUsageDao; @@ -86,11 +89,11 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi private QuotaBalanceDao _quotaBalanceDao; @Inject private QuotaResponseBuilder _respBldr; + @Inject + private ProjectManager projectMgr; private TimeZone _usageTimezone; - private boolean jsInterpretationEnabled = false; - public QuotaServiceImpl() { super(); } @@ -102,8 +105,6 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT"); _usageTimezone = TimeZone.getTimeZone(timeZoneStr); - jsInterpretationEnabled = ManagementService.JsInterpretationEnabled.value(); - return true; } @@ -292,9 +293,4 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi _quotaAcc.updateQuotaAccount(accountId, acc); } } - - @Override - public boolean isJsInterpretationEnabled() { - return jsInterpretationEnabled; - } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index 3629bf2e3fe..ea88a106b84 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -16,13 +16,11 @@ // under the License. package org.apache.cloudstack.api.response; -import java.lang.reflect.Field; import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; -import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -31,6 +29,7 @@ import java.util.Set; import java.util.HashSet; import java.util.function.Consumer; +import com.cloud.domain.Domain; import com.cloud.domain.DomainVO; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.PermissionDeniedException; @@ -43,10 +42,10 @@ import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.discovery.ApiDiscoveryService; -import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.jsinterpreter.JsInterpreterHelper; import org.apache.cloudstack.quota.QuotaService; import org.apache.cloudstack.quota.QuotaStatement; @@ -79,7 +78,10 @@ import org.junit.runner.RunWith; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.Account; @@ -89,8 +91,7 @@ import com.cloud.user.dao.UserDao; import com.cloud.user.User; import junit.framework.TestCase; -import org.mockito.Spy; -import org.mockito.junit.MockitoJUnitRunner; + @RunWith(MockitoJUnitRunner.class) public class QuotaResponseBuilderImplTest extends TestCase { @@ -153,7 +154,7 @@ public class QuotaResponseBuilderImplTest extends TestCase { Account accountMock; @Mock - DomainVO domainVOMock; + DomainVO domainVoMock; @Mock QuotaConfigureEmailCmd quotaConfigureEmailCmdMock; @@ -161,6 +162,9 @@ public class QuotaResponseBuilderImplTest extends TestCase { @Mock QuotaAccountVO quotaAccountVOMock; + @Mock + CallContext callContextMock; + @Mock QuotaEmailTemplatesVO quotaEmailTemplatesVoMock; @@ -184,17 +188,8 @@ public class QuotaResponseBuilderImplTest extends TestCase { CallContext.register(callerUserMock, callerAccountMock); } - private void overrideDefaultQuotaEnabledConfigValue(final Object value) throws IllegalAccessException, NoSuchFieldException { - Field f = ConfigKey.class.getDeclaredField("_defaultValue"); - f.setAccessible(true); - f.set(QuotaConfig.QuotaAccountEnabled, value); - } - - private Calendar[] createPeriodForQuotaSummary() { - final Calendar calendar = Calendar.getInstance(); - calendar.set(Calendar.HOUR, 0); - return new Calendar[] {calendar, calendar}; - } + @Mock + Pair, Integer> quotaSummaryResponseMock1, quotaSummaryResponseMock2; @Mock QuotaValidateActivationRuleCmd quotaValidateActivationRuleCmdMock = Mockito.mock(QuotaValidateActivationRuleCmd.class); @@ -466,36 +461,6 @@ public class QuotaResponseBuilderImplTest extends TestCase { Mockito.verify(quotaTariffVoMock).setRemoved(Mockito.any(Date.class)); } - @Test - public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsDisabledShouldReturnFalse() throws NoSuchFieldException, IllegalAccessException { - Calendar[] period = createPeriodForQuotaSummary(); - overrideDefaultQuotaEnabledConfigValue("false"); - - Mockito.doReturn(period).when(quotaStatementMock).getCurrentStatementTime(); - Mockito.doReturn(domainVOMock).when(domainDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(BigDecimal.ZERO).when(quotaBalanceDaoMock).lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class)); - Mockito.doReturn(BigDecimal.ZERO).when(quotaUsageDaoMock).findTotalQuotaUsage(Mockito.anyLong(), Mockito.anyLong(), Mockito.isNull(), Mockito.any(Date.class), Mockito.any(Date.class)); - - QuotaSummaryResponse quotaSummaryResponse = quotaResponseBuilderSpy.getQuotaSummaryResponse(accountMock); - - assertFalse(quotaSummaryResponse.getQuotaEnabled()); - } - - @Test - public void getQuotaSummaryResponseTestAccountIsNotNullQuotaIsEnabledShouldReturnTrue() throws NoSuchFieldException, IllegalAccessException { - Calendar[] period = createPeriodForQuotaSummary(); - overrideDefaultQuotaEnabledConfigValue("true"); - - Mockito.doReturn(period).when(quotaStatementMock).getCurrentStatementTime(); - Mockito.doReturn(domainVOMock).when(domainDaoMock).findById(Mockito.anyLong()); - Mockito.doReturn(BigDecimal.ZERO).when(quotaBalanceDaoMock).lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class)); - Mockito.doReturn(BigDecimal.ZERO).when(quotaUsageDaoMock).findTotalQuotaUsage(Mockito.anyLong(), Mockito.anyLong(), Mockito.isNull(), Mockito.any(Date.class), Mockito.any(Date.class)); - - QuotaSummaryResponse quotaSummaryResponse = quotaResponseBuilderSpy.getQuotaSummaryResponse(accountMock); - - assertTrue(quotaSummaryResponse.getQuotaEnabled()); - } - @Test public void filterSupportedTypesTestReturnWhenQuotaTypeDoesNotMatch() throws NoSuchFieldException { List> variables = new ArrayList<>(); @@ -576,6 +541,63 @@ public class QuotaResponseBuilderImplTest extends TestCase { quotaResponseBuilderSpy.validateQuotaConfigureEmailCmdParameters(quotaConfigureEmailCmdMock); } + @Test + public void createQuotaSummaryResponseTestNotListAllAndAllAccountTypesReturnsSingleRecord() { + QuotaSummaryCmd cmd = new QuotaSummaryCmd(); + + try(MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + + Mockito.doReturn(quotaSummaryResponseMock1).when(quotaResponseBuilderSpy).getQuotaSummaryResponse(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + + for (Account.Type type : Account.Type.values()) { + Mockito.doReturn(type).when(accountMock).getType(); + + Pair, Integer> result = quotaResponseBuilderSpy.createQuotaSummaryResponse(cmd); + Assert.assertEquals(quotaSummaryResponseMock1, result); + } + + Mockito.verify(quotaResponseBuilderSpy, Mockito.times(Account.Type.values().length)).getQuotaSummaryResponse(Mockito.any(), Mockito.any(), Mockito.any(), + Mockito.any()); + }; + } + + @Test + public void getDomainPathByDomainIdForDomainAdminTestAccountNotDomainAdminReturnsNull() { + for (Account.Type type : Account.Type.values()) { + if (Account.Type.DOMAIN_ADMIN.equals(type)) { + continue; + } + + Mockito.doReturn(type).when(accountMock).getType(); + Assert.assertNull(quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock)); + } + } + + @Test(expected = InvalidParameterValueException.class) + public void getDomainPathByDomainIdForDomainAdminTestDomainFromCallerIsNullThrowsInvalidParameterValueException() { + Mockito.doReturn(Account.Type.DOMAIN_ADMIN).when(accountMock).getType(); + Mockito.doReturn(null).when(domainDaoMock).findById(Mockito.anyLong()); + Mockito.lenient().doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + + quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock); + } + + @Test + public void getDomainPathByDomainIdForDomainAdminTestDomainFromCallerIsNotNullReturnsPath() { + String expected = "/test/"; + + Mockito.doReturn(Account.Type.DOMAIN_ADMIN).when(accountMock).getType(); + Mockito.doReturn(domainVoMock).when(domainDaoMock).findById(Mockito.anyLong()); + Mockito.doNothing().when(accountManagerMock).checkAccess(Mockito.any(Account.class), Mockito.any(Domain.class)); + Mockito.doReturn(expected).when(domainVoMock).getPath(); + + String result = quotaResponseBuilderSpy.getDomainPathByDomainIdForDomainAdmin(accountMock); + Assert.assertEquals(expected, result); + } + @Test public void getQuotaEmailConfigurationVoTestTemplateNameIsNull() { Mockito.doReturn(null).when(quotaConfigureEmailCmdMock).getTemplateName(); diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 19e756d1d97..259264f3b0e 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.quota; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; import junit.framework.TestCase; @@ -33,8 +34,10 @@ import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import javax.naming.ConfigurationException; @@ -48,7 +51,7 @@ import java.util.List; public class QuotaServiceImplTest extends TestCase { @Mock - AccountDao accountDao; + AccountDao accountDaoMock; @Mock QuotaAccountDao quotaAcc; @Mock @@ -61,8 +64,13 @@ public class QuotaServiceImplTest extends TestCase { QuotaBalanceDao quotaBalanceDao; @Mock QuotaResponseBuilder respBldr; + @Mock + private AccountVO accountVoMock; + + @Spy + @InjectMocks + QuotaServiceImpl quotaServiceImplSpy; - QuotaServiceImpl quotaService = new QuotaServiceImpl(); @Before public void setup() throws IllegalAccessException, NoSuchFieldException, ConfigurationException { @@ -71,34 +79,34 @@ public class QuotaServiceImplTest extends TestCase { Field accountDaoField = QuotaServiceImpl.class.getDeclaredField("_accountDao"); accountDaoField.setAccessible(true); - accountDaoField.set(quotaService, accountDao); + accountDaoField.set(quotaServiceImplSpy, accountDaoMock); Field quotaAccountDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaAcc"); quotaAccountDaoField.setAccessible(true); - quotaAccountDaoField.set(quotaService, quotaAcc); + quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaService, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); - domainDaoField.set(quotaService, domainDao); + domainDaoField.set(quotaServiceImplSpy, domainDao); Field configDaoField = QuotaServiceImpl.class.getDeclaredField("_configDao"); configDaoField.setAccessible(true); - configDaoField.set(quotaService, configDao); + configDaoField.set(quotaServiceImplSpy, configDao); Field balanceDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaBalanceDao"); balanceDaoField.setAccessible(true); - balanceDaoField.set(quotaService, quotaBalanceDao); + balanceDaoField.set(quotaServiceImplSpy, quotaBalanceDao); Field QuotaResponseBuilderField = QuotaServiceImpl.class.getDeclaredField("_respBldr"); QuotaResponseBuilderField.setAccessible(true); - QuotaResponseBuilderField.set(quotaService, respBldr); + QuotaResponseBuilderField.set(quotaServiceImplSpy, respBldr); Mockito.when(configDao.getValue(Mockito.eq(Config.UsageAggregationTimezone.toString()))).thenReturn("IST"); - quotaService.configure("randomName", null); + quotaServiceImplSpy.configure("randomName", null); } @Test @@ -120,9 +128,9 @@ public class QuotaServiceImplTest extends TestCase { Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); // with enddate - assertTrue(quotaService.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); + assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); // without enddate - assertTrue(quotaService.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); + assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); } @Test @@ -133,7 +141,7 @@ public class QuotaServiceImplTest extends TestCase { final Date startDate = new DateTime().minusDays(2).toDate(); final Date endDate = new Date(); - quotaService.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); + quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); } @@ -142,13 +150,13 @@ public class QuotaServiceImplTest extends TestCase { // existing account QuotaAccountVO quotaAccountVO = new QuotaAccountVO(); Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(quotaAccountVO); - quotaService.setLockAccount(2L, true); + quotaServiceImplSpy.setLockAccount(2L, true); Mockito.verify(quotaAcc, Mockito.times(0)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); Mockito.verify(quotaAcc, Mockito.times(1)).updateQuotaAccount(Mockito.anyLong(), Mockito.any(QuotaAccountVO.class)); // new account Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(null); - quotaService.setLockAccount(2L, true); + quotaServiceImplSpy.setLockAccount(2L, true); Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } @@ -160,13 +168,14 @@ public class QuotaServiceImplTest extends TestCase { // existing account setting QuotaAccountVO quotaAccountVO = new QuotaAccountVO(); Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(quotaAccountVO); - quotaService.setMinBalance(accountId, balance); + quotaServiceImplSpy.setMinBalance(accountId, balance); Mockito.verify(quotaAcc, Mockito.times(0)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); Mockito.verify(quotaAcc, Mockito.times(1)).updateQuotaAccount(Mockito.anyLong(), Mockito.any(QuotaAccountVO.class)); // no account with limit set Mockito.when(quotaAcc.findByIdQuotaAccount(Mockito.anyLong())).thenReturn(null); - quotaService.setMinBalance(accountId, balance); + quotaServiceImplSpy.setMinBalance(accountId, balance); Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + } diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java index 940897de3c9..c6c38a39809 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java @@ -106,7 +106,6 @@ public class BareMetalTemplateAdapter extends TemplateAdapterBase implements Tem } } - _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); return template; } diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config index b66258ad9f6..e96b6d8b8c5 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/AgentShell/App.config @@ -35,7 +35,7 @@ - + diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/App.config b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/App.config index 3c4f7066082..c5025fb267c 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/App.config +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/HypervResource/App.config @@ -1,5 +1,5 @@ - diff --git a/plugins/hypervisors/hyperv/DotNet/ServerResource/ServerResource.Tests/App.config b/plugins/hypervisors/hyperv/DotNet/ServerResource/ServerResource.Tests/App.config index ddf1da0bcf6..184c44a5f27 100644 --- a/plugins/hypervisors/hyperv/DotNet/ServerResource/ServerResource.Tests/App.config +++ b/plugins/hypervisors/hyperv/DotNet/ServerResource/ServerResource.Tests/App.config @@ -1,5 +1,5 @@ - diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index da60f6fd717..e19c3437ba5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -275,6 +275,7 @@ public class BridgeVifDriver extends VifDriverBase { if (nic.getPxeDisable()) { intf.setPxeDisable(true); } + intf.setLinkStateUp(nic.isEnabled()); return intf; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 896426addca..e9a7ac8951c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -35,7 +35,6 @@ import com.cloud.agent.properties.AgentPropertiesFileHandler; public class KVMHABase { protected Logger logger = LogManager.getLogger(getClass()); private long _timeout = 60000; /* 1 minutes */ - protected static String s_heartBeatPath; protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index cf407bfc08a..aa868ff1d3f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -18,7 +18,7 @@ package com.cloud.hypervisor.kvm.resource; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; -import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.utils.script.Script; import org.libvirt.Connect; import org.libvirt.LibvirtException; @@ -39,20 +39,15 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { private final String hostPrivateIp; - public KVMHAMonitor(HAStoragePool pool, String host, String scriptPath) { + public KVMHAMonitor(HAStoragePool pool, String host) { if (pool != null) { storagePool.put(pool.getPoolUUID(), pool); } hostPrivateIp = host; - configureHeartBeatPath(scriptPath); rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } - private static synchronized void configureHeartBeatPath(String scriptPath) { - KVMHABase.s_heartBeatPath = scriptPath; - } - public void addStoragePool(HAStoragePool pool) { synchronized (storagePool) { storagePool.put(pool.getPoolUUID(), pool); @@ -86,8 +81,8 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { Set removedPools = new HashSet<>(); for (String uuid : storagePool.keySet()) { HAStoragePool primaryStoragePool = storagePool.get(uuid); - if (primaryStoragePool.getPool().getType() == StoragePoolType.NetworkFilesystem) { - checkForNotExistingPools(removedPools, uuid); + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(primaryStoragePool.getPool().getType())) { + checkForNotExistingLibvirtStoragePools(removedPools, uuid); if (removedPools.contains(uuid)) { continue; } @@ -127,7 +122,7 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { return result; } - private void checkForNotExistingPools(Set removedPools, String uuid) { + private void checkForNotExistingLibvirtStoragePools(Set removedPools, String uuid) { try { Connect conn = LibvirtConnection.getConnection(); StoragePool storage = conn.storagePoolLookupByUUIDString(uuid); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 55bab118ad0..67ea7a1d0dc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -1065,11 +1065,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv throw new ConfigurationException("Unable to find patch.sh"); } - heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (heartBeatPath == null) { - throw new ConfigurationException("Unable to find kvmheartbeat.sh"); - } - createVmPath = Script.findScript(storageScriptsDir, "createvm.sh"); if (createVmPath == null) { throw new ConfigurationException("Unable to find the createvm.sh"); @@ -1332,7 +1327,7 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final String[] info = NetUtils.getNetworkParams(privateNic); - kvmhaMonitor = new KVMHAMonitor(null, info[0], heartBeatPath); + kvmhaMonitor = new KVMHAMonitor(null, info[0]); final Thread ha = new Thread(kvmhaMonitor); ha.start(); @@ -4851,6 +4846,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv } } + public InterfaceDef getInterface(final Connect conn, final String vmName, final String macAddress) { + List interfaces = getInterfaces(conn, vmName); + return interfaces.stream().filter(interfaceDef -> interfaceDef.getMacAddress().equals(macAddress)) + .findFirst().orElseThrow(() -> new CloudRuntimeException(String.format("Unable to find NIC with MAC address %s.", macAddress))); + } + public List getDisks(final Connect conn, final String vmName) { final LibvirtDomainXMLParser parser = new LibvirtDomainXMLParser(); Domain dm = null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java index 08086859fb7..d3765c01ccb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java @@ -79,28 +79,47 @@ public class LibvirtGpuDef { gpuBuilder.append(" \n"); gpuBuilder.append(" \n"); - // Parse the bus address (e.g., 00:02.0) into domain, bus, slot, function - String domain = "0x0000"; - String bus = "0x00"; - String slot = "0x00"; - String function = "0x0"; + // Parse the bus address into domain, bus, slot, function. Two input formats are accepted: + // - "dddd:bb:ss.f" full PCI address with domain (e.g. 0000:00:02.0) + // - "bb:ss.f" legacy short BDF; domain defaults to 0000 + // Each segment is parsed as a hex integer and formatted with fixed widths + // (domain: 4 hex digits, bus/slot: 2 hex digits, function: 1 hex digit) to + // produce canonical libvirt XML values regardless of input casing or padding. + int domainVal = 0, busVal = 0, slotVal = 0, funcVal = 0; if (busAddress != null && !busAddress.isEmpty()) { String[] parts = busAddress.split(":"); - if (parts.length > 1) { - bus = "0x" + parts[0]; - String[] slotFunctionParts = parts[1].split("\\."); - if (slotFunctionParts.length > 0) { - slot = "0x" + slotFunctionParts[0]; - if (slotFunctionParts.length > 1) { - function = "0x" + slotFunctionParts[1].trim(); - } + try { + String slotFunction; + if (parts.length == 3) { + domainVal = Integer.parseInt(parts[0], 16); + busVal = Integer.parseInt(parts[1], 16); + slotFunction = parts[2]; + } else if (parts.length == 2) { + busVal = Integer.parseInt(parts[0], 16); + slotFunction = parts[1]; + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); } + String[] sf = slotFunction.split("\\."); + if (sf.length == 2) { + slotVal = Integer.parseInt(sf[0], 16); + funcVal = Integer.parseInt(sf[1].trim(), 16); + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'", e); } } + String domain = String.format("0x%04x", domainVal); + String bus = String.format("0x%02x", busVal); + String slot = String.format("0x%02x", slotVal); + String function = String.format("0x%x", funcVal); + gpuBuilder.append("

\n"); + .append(slot).append("' function='").append(function).append("'/>\n"); gpuBuilder.append(" \n"); gpuBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index a708d441be5..d3f537dc917 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -48,7 +48,7 @@ public final class LibvirtCheckVMActivityOnStoragePoolCommandWrapper extends Com KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid()); - if (primaryPool.isPoolSupportHA()){ + if (primaryPool.isPoolSupportHA()) { final HAStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java index 99005755cbb..aa6a37b6561 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java @@ -45,11 +45,10 @@ public final class LibvirtCheckVirtualMachineCommandWrapper extends CommandWrapp Integer vncPort = null; if (state == PowerState.PowerOn) { vncPort = libvirtComputingResource.getVncPort(conn, command.getVmName()); - } - - Domain vm = conn.domainLookupByName(command.getVmName()); - if (state == PowerState.PowerOn && DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { - return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + Domain vm = conn.domainLookupByName(command.getVmName()); + if (DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { + return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + } } return new CheckVirtualMachineAnswer(command, state, vncPort); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 55225ba086c..6788516df74 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -47,7 +47,10 @@ import java.util.Map; @ResourceWrapper(handles = CheckVolumeCommand.class) public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper { - private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem); + private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( + Storage.StoragePoolType.Filesystem, + Storage.StoragePoolType.NetworkFilesystem, + Storage.StoragePoolType.SharedMountPoint); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java index 114b27d3a5b..c0887415c65 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetRemoteVmsCommandWrapper.java @@ -22,6 +22,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import com.cloud.agent.api.Answer; import com.cloud.agent.api.GetRemoteVmsAnswer; import com.cloud.agent.api.GetRemoteVmsCommand; +import com.cloud.hypervisor.Hypervisor; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.resource.LibvirtConnection; import com.cloud.hypervisor.kvm.resource.LibvirtDomainXMLParser; @@ -97,6 +98,7 @@ public final class LibvirtGetRemoteVmsCommandWrapper extends CommandWrapper { static final List STORAGE_POOL_TYPES_SUPPORTED_BY_QEMU_IMG = Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem, StoragePoolType.RBD); + StoragePoolType.Filesystem, StoragePoolType.RBD, StoragePoolType.SharedMountPoint); @Override public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 714e3844b34..22dbfbdd67a 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -41,9 +41,9 @@ import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; -import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; import java.util.Locale; @@ -56,10 +56,25 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper restoreVolumePools = command.getRestoreVolumePools(); List restoreVolumePaths = command.getRestoreVolumePaths(); Integer mountTimeout = command.getMountTimeout() * 1000; - int timeout = command.getWait(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : serverResource.getCmdsTimeout(); KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); List backupFiles = command.getBackupFiles(); @@ -84,9 +99,9 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(vmName, command.getVmState()), mountDirectory, timeout); } else if (Boolean.TRUE.equals(vmExists)) { restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); @@ -143,7 +158,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout) { String bkpPath; String volumeUuid; try { bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); - volumeUuid = volumePath.substring(volumePath.lastIndexOf(File.separator) + 1); + volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); verifyBackupFile(bkpPath, volumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true)) { + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true, size)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid)); } @@ -247,42 +262,66 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper volumePools = command.getVolumePools(); final List volumePaths = command.getVolumePaths(); KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout(); List diskPaths = new ArrayList<>(); if (Objects.nonNull(volumePaths)) { @@ -81,7 +82,7 @@ public class LibvirtTakeBackupCommandWrapper extends CommandWrapper result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + Pair result = Script.executePipedCommands(commands, timeout); if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateVmNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateVmNicCommandWrapper.java new file mode 100644 index 00000000000..85f00e1dbd4 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUpdateVmNicCommandWrapper.java @@ -0,0 +1,67 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.UpdateVmNicAnswer; +import com.cloud.agent.api.UpdateVmNicCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; + +@ResourceWrapper(handles = UpdateVmNicCommand.class) +public final class LibvirtUpdateVmNicCommandWrapper extends CommandWrapper { + + @Override + public Answer execute(UpdateVmNicCommand command, LibvirtComputingResource libvirtComputingResource) { + String nicMacAddress = command.getNicMacAddress(); + String vmName = command.getVmName(); + boolean isEnabled = command.isEnabled(); + + Domain vm = null; + try { + final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper(); + final Connect conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName); + vm = libvirtComputingResource.getDomain(conn, vmName); + + final InterfaceDef nic = libvirtComputingResource.getInterface(conn, vmName, nicMacAddress); + nic.setLinkStateUp(isEnabled); + vm.updateDeviceFlags(nic.toString(), Domain.DeviceModifyFlags.LIVE); + + return new UpdateVmNicAnswer(command, true, "success"); + } catch (final LibvirtException e) { + final String msg = String.format(" Update NIC failed due to %s.", e); + logger.warn(msg, e); + return new UpdateVmNicAnswer(command, false, msg); + } finally { + if (vm != null) { + try { + vm.free(); + } catch (final LibvirtException l) { + logger.trace("Ignoring libvirt error.", l); + } + } + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 6665cf625e2..35cc864268c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -289,6 +289,7 @@ public class KVMStoragePoolManager { if (pool instanceof LibvirtStoragePool) { addPoolDetails(uuid, (LibvirtStoragePool) pool); + ((LibvirtStoragePool) pool).setType(type); } return pool; @@ -390,6 +391,9 @@ public class KVMStoragePoolManager { private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); + if (pool instanceof LibvirtStoragePool) { + ((LibvirtStoragePool) pool).setType(type); + } // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 030d9747d6c..6e03b84d20c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -185,6 +185,8 @@ public class KVMStorageProcessor implements StorageProcessor { private int incrementalSnapshotTimeout; + private int incrementalSnapshotRetryRebaseWait; + private static final String CHECKPOINT_XML_TEMP_DIR = "/tmp/cloudstack/checkpointXMLs"; private static final String BACKUP_XML_TEMP_DIR = "/tmp/cloudstack/backupXMLs"; @@ -252,6 +254,7 @@ public class KVMStorageProcessor implements StorageProcessor { _cmdsTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CMDS_TIMEOUT) * 1000; incrementalSnapshotTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000; + incrementalSnapshotRetryRebaseWait = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT) * 1000; return true; } @@ -2093,8 +2096,25 @@ public class KVMStorageProcessor implements StorageProcessor { QemuImg qemuImg = new QemuImg(wait); qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); } catch (LibvirtException | QemuImgException e) { - logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); - throw new CloudRuntimeException(e); + if (!StringUtils.contains(e.getMessage(), "Is another process using the image")) { + logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + retryRebase(snapshotName, wait, e, snapshotFile, parentSnapshotFile); + } + } + + private void retryRebase(String snapshotName, int wait, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) { + logger.warn("Libvirt still has not released the lock, will wait [{}] milliseconds and try again later.", incrementalSnapshotRetryRebaseWait); + try { + Thread.sleep(incrementalSnapshotRetryRebaseWait); + QemuImg qemuImg = new QemuImg(wait); + qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); + } catch (LibvirtException | QemuImgException | InterruptedException ex) { + logger.error("Unable to rebase snapshot [{}].", snapshotName, ex); + CloudRuntimeException cre = new CloudRuntimeException(ex); + cre.addSuppressed(e); + throw cre; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index ab39f7bc6ff..45c22d3ac75 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import com.cloud.agent.api.to.HostTO; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; @@ -320,13 +321,24 @@ public class LibvirtStoragePool implements KVMStoragePool { @Override public boolean isPoolSupportHA() { - return type == StoragePoolType.NetworkFilesystem; + return HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type); } public String getHearthBeatPath() { - if (type == StoragePoolType.NetworkFilesystem) { + if (StoragePoolType.NetworkFilesystem.equals(type)) { String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - return Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); + String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); + if (scriptPath == null) { + throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); + } + return scriptPath; + } else if (StoragePoolType.SharedMountPoint.equals(type)) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); + if (scriptPath == null) { + throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); + } + return scriptPath; } return null; } @@ -410,4 +422,8 @@ public class LibvirtStoragePool implements KVMStoragePool { return true; } } + + public void setType(StoragePoolType type) { + this.type = type; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index e336e0e5a54..82bc35f009e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Predicate; import com.cloud.agent.api.PrepareStorageClientCommand; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; @@ -644,18 +645,30 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { // Assuming SDC service is started, add mdms String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS); String[] mdmAddresses = mdms.split(","); - if (mdmAddresses.length > 0) { - if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client"); - } - - ScaleIOUtil.addMdms(mdmAddresses); - if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(false, null, "Failed to add MDMs"); + // remove MDMs already present in the config and added to the SDC + String[] mdmAddressesToAdd = Arrays.stream(mdmAddresses) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + // if all MDMs are already in the config and added to the SDC + if (mdmAddressesToAdd.length < 1 && mdmAddresses.length > 0) { + String msg = String.format("MDMs %s of the storage pool %s are already added", mdms, uuid); + logger.debug(msg); + return new Ternary<>(true, getSDCDetails(details), msg); + } else if (mdmAddressesToAdd.length > 0) { + ScaleIOUtil.addMdms(mdmAddressesToAdd); + String[] missingMdmAddresses = Arrays.stream(mdmAddressesToAdd) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + if (missingMdmAddresses.length > 0) { + String msg = String.format("Failed to add MDMs %s of the storage pool %s", String.join(", ", missingMdmAddresses), uuid); + logger.debug(msg); + return new Ternary<>(false, null, msg); } else { - logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid)); + logger.debug("MDMs {} of the storage pool {} are added", mdmAddressesToAdd, uuid); applyMdmsChangeWaitTime(details); } + } else { + return new Ternary<>(false, getSDCDetails(details), "No MDM addresses provided"); } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index cde87fd9384..b9629524007 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -7203,4 +7203,35 @@ public class LibvirtComputingResourceTest { libvirtComputingResourceSpy.defineDiskForDefaultPoolType(diskDef, volume, false, false, false, physicalDisk, DEV_ID, DISK_BUS_TYPE, DISK_BUS_TYPE_DATA, null); Mockito.verify(diskDef).defFileBasedDisk(PHYSICAL_DISK_PATH, DEV_ID, DISK_BUS_TYPE_DATA, DiskDef.DiskFmtType.QCOW2); } + + @Test + public void getInterfaceTestValidMacAddressReturnInterface() { + String macAddress = "a0:90:27:a9:9e:62"; + final String vmName = "Test"; + final InterfaceDef interfaceDef = Mockito.mock(InterfaceDef.class); + final List interfaces = new ArrayList<>(); + interfaces.add(interfaceDef); + + Mockito.doReturn(macAddress).when(interfaceDef).getMacAddress(); + Mockito.doReturn(interfaces).when(libvirtComputingResourceSpy).getInterfaces(Mockito.any(), Mockito.anyString()); + + InterfaceDef result = libvirtComputingResourceSpy.getInterface(connMock, vmName, macAddress); + + Assert.assertNotNull(result); + } + + @Test(expected = CloudRuntimeException.class) + public void getInterfaceTestInvalidMacAddressThrowCloudRuntimeException() { + String invalidMacAddress = "ea:57:5d:f1:64:05"; + String macAddress = "a0:90:27:a9:9e:62"; + final String vmName = "Test"; + final InterfaceDef interfaceDef = Mockito.mock(InterfaceDef.class); + final List interfaces = new ArrayList<>(); + interfaces.add(interfaceDef); + + Mockito.doReturn(macAddress).when(interfaceDef).getMacAddress(); + Mockito.doReturn(interfaces).when(libvirtComputingResourceSpy).getInterfaces(Mockito.any(), Mockito.anyString()); + + libvirtComputingResourceSpy.getInterface(connMock, vmName, invalidMacAddress); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java index 0060e1d7ed4..e6f63e852f8 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java @@ -115,6 +115,145 @@ public class LibvirtGpuDefTest extends TestCase { assertTrue(gpuXml.contains("")); } + @Test + public void testGpuDef_withFullPciAddressDomainZero() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0000:00:02.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0001:65:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withNvidiaStyleEightDigitDomain() { + // nvidia-smi reports PCI addresses with an 8-digit domain (e.g. "00000001:af:00.1"). + // generatePciXml must normalize it to the canonical 4-digit form "0x0001". + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "00000001:af:00.1", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressVfNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo vfGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "VF-Profile", + "VF-Profile", + "0002:81:00.3", + "10de", + "NVIDIA Corporation", + "1eb8", + "Tesla T4" + ); + gpuDef.defGpu(vfGpuInfo); + + String gpuXml = gpuDef.toString(); + + // Non-passthrough NVIDIA VFs should be unmanaged + assertTrue(gpuXml.contains("")); + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withLegacyShortBdfDefaultsDomainToZero() { + // Backward compatibility: short BDF with no domain segment must still + // produce a valid libvirt address with domain 0x0000. + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "af:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withInvalidBusAddressThrows() { + String[] invalidAddresses = { + "notahex:00.0", // non-hex bus + "gg:00:02.0", // non-hex domain + "00:02:03:04", // too many colon-separated parts + "00", // missing slot/function + "00:02", // missing function (no dot) + "00:02.0.1", // extra dot in ss.f + }; + for (String addr : invalidAddresses) { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo info = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + addr, + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(info); + try { + String ignored = gpuDef.toString(); + fail("Expected IllegalArgumentException for address: " + addr + " but got: " + ignored); + } catch (IllegalArgumentException e) { + assertTrue("Exception message should contain the bad address", + e.getMessage().contains(addr)); + } + } + } + @Test public void testGpuDef_withNullDeviceType() { LibvirtGpuDef gpuDef = new LibvirtGpuDef(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java new file mode 100644 index 00000000000..a06a9c50bc1 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; +import org.libvirt.DomainInfo.DomainState; +import org.libvirt.LibvirtException; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.CheckVirtualMachineAnswer; +import com.cloud.agent.api.CheckVirtualMachineCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.vm.VirtualMachine.PowerState; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckVirtualMachineCommandWrapperTest { + + private static final String VM_NAME = "i-2-3-VM"; + + @Mock + private LibvirtComputingResource libvirtComputingResource; + @Mock + private LibvirtUtilitiesHelper libvirtUtilitiesHelper; + @Mock + private Connect conn; + @Mock + private Domain domain; + + private LibvirtCheckVirtualMachineCommandWrapper wrapper; + private CheckVirtualMachineCommand command; + + @Before + public void setUp() throws LibvirtException { + wrapper = new LibvirtCheckVirtualMachineCommandWrapper(); + command = new CheckVirtualMachineCommand(VM_NAME); + + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnectionByVmName(VM_NAME)).thenReturn(conn); + } + + @Test + public void testExecuteVmPoweredOnReturnsStateAndVncPort() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_RUNNING; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5900); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOn, answer.getState()); + assertEquals(Integer.valueOf(5900), answer.getVncPort()); + } + + @Test + public void testExecuteVmPausedReturnsPowerUnknown() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_PAUSED; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5901); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerUnknown, answer.getState()); + assertEquals(Integer.valueOf(5901), answer.getVncPort()); + } + + @Test + public void testExecuteVmPoweredOffReturnsStateWithNullVncPort() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOff); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOff, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteVmStateUnknownReturnsStateWithNullVncPort() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerUnknown); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerUnknown, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteVmPoweredOnWithNullVncPort() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_RUNNING; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(null); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOn, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteLibvirtExceptionOnGetConnectionReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("Connection refused"); + when(libvirtUtilitiesHelper.getConnectionByVmName(VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("Connection refused", answer.getDetails()); + } + + @Test + public void testExecuteLibvirtExceptionOnGetVncPortReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("VNC port error"); + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("VNC port error", answer.getDetails()); + } + + @Test + public void testExecuteLibvirtExceptionOnDomainLookupReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("Domain not found"); + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5900); + when(conn.domainLookupByName(VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("Domain not found", answer.getDetails()); + } + + @Test + public void testExecuteCallsGetLibvirtUtilitiesHelper() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOff); + + wrapper.execute(command, libvirtComputingResource); + + verify(libvirtComputingResource).getLibvirtUtilitiesHelper(); + verify(libvirtUtilitiesHelper).getConnectionByVmName(VM_NAME); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java index d72dc0d8ac3..ef6b5c08189 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapperTest.java @@ -18,6 +18,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.storage.Storage; import com.cloud.utils.script.Script; import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.backup.BackupAnswer; @@ -66,7 +67,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -109,6 +113,7 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.isVmExists()).thenReturn(true); when(command.getDiskType()).thenReturn("root"); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupVolumesUUIDs()).thenReturn(Arrays.asList("volume-123")); @@ -148,6 +153,7 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.isVmExists()).thenReturn(false); when(command.getDiskType()).thenReturn("root"); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -185,7 +191,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("username=user,password=pass"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -226,7 +235,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { lenient().when(command.getMountOptions()).thenReturn("rw"); lenient().when(command.isVmExists()).thenReturn(null); lenient().when(command.getDiskType()).thenReturn("root"); + lenient().when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + lenient().when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + lenient().when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); lenient().when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -262,7 +274,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -308,7 +323,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -356,7 +374,10 @@ public class LibvirtRestoreBackupCommandWrapperTest { when(command.getMountOptions()).thenReturn("rw"); when(command.isVmExists()).thenReturn(null); when(command.getDiskType()).thenReturn("root"); + when(command.getRestoreVolumeSizes()).thenReturn(Arrays.asList(1024L)); + when(command.getWait()).thenReturn(60); PrimaryDataStoreTO primaryDataStore = Mockito.mock(PrimaryDataStoreTO.class); + when(primaryDataStore.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); when(command.getRestoreVolumePools()).thenReturn(Arrays.asList(primaryDataStore)); when(command.getRestoreVolumePaths()).thenReturn(Arrays.asList("/var/lib/libvirt/images/volume-123")); when(command.getBackupFiles()).thenReturn(Arrays.asList("volume-123")); @@ -370,7 +391,15 @@ public class LibvirtRestoreBackupCommandWrapperTest { try (MockedStatic -