diff --git a/.asf.yaml b/.asf.yaml index a19e3a60a2b..3b2f7691d13 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -50,10 +50,13 @@ github: rebase: false collaborators: - - gpordeus + - ingox + - gp-santos - erikbocks - Imvedansh - Damans227 + - jmsperu + - GaOrtiga protected_branches: ~ diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a0602871ef1..689275cff11 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,6 +17,7 @@ /plugins/storage/volume/linstor @rp- /plugins/storage/volume/storpool @slavkap +/plugins/storage/volume/ontap @rajiv1 @sandeeplocharla @piyush5 @suryag .pre-commit-config.yaml @jbampton /.github/linters/ @jbampton diff --git a/.github/actions/install-nonoss/action.yml b/.github/actions/install-nonoss/action.yml new file mode 100644 index 00000000000..39a03213c29 --- /dev/null +++ b/.github/actions/install-nonoss/action.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Install CloudStack Non-OSS' +description: 'Clones and installs the shapeblue/cloudstack-nonoss repository.' + +runs: + using: "composite" + steps: + - name: Install cloudstack-nonoss + shell: bash + run: | + git clone --depth 1 https://github.com/shapeblue/cloudstack-nonoss.git nonoss + cd nonoss + bash -x install-non-oss.sh + cd .. + rm -fr nonoss diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml new file mode 100644 index 00000000000..0f842522924 --- /dev/null +++ b/.github/actions/setup-env/action.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: 'Setup CloudStack Environment' +description: 'Sets up JDK (with Maven cache), optionally Python, and optionally APT build dependencies for CloudStack.' + +inputs: + java-version: + description: 'The JDK version to use' + required: false + default: '17' + install-python: + description: 'Whether to install Python 3.10' + required: false + default: 'false' + install-apt-deps: + description: 'Whether to install CloudStack APT build dependencies' + required: false + default: 'false' + +runs: + using: "composite" + steps: + - name: Set up JDK ${{ inputs.java-version }} + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + java-version: ${{ inputs.java-version }} + distribution: 'adopt' + architecture: x64 + cache: 'maven' + + - name: Set up Python + if: ${{ inputs.install-python == 'true' }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.10' + architecture: x64 + + - name: Install Build Dependencies + if: ${{ inputs.install-apt-deps == 'true' }} + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4c33a131343..4eb508f5f6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,40 +16,27 @@ # under the License. name: Build - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' + persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v6 + - name: Setup Environment + uses: ./.github/actions/setup-env with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Env details run: | uname -a @@ -60,9 +47,8 @@ jobs: free -m nproc git status - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Noredist Build run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -B -P developer,systemvm -Dsimulator -Dnoredist clean install -T$(nproc) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df60179ceb5..e9d03bf1209 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,21 +16,56 @@ # under the License. name: Simulator CI - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: if: github.repository == 'apache/cloudstack' runs-on: ubuntu-24.04 - + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + with: + install-python: 'true' + install-apt-deps: 'true' + - name: Env details + run: | + uname -a + whoami + javac -version + mvn -v + python3 --version + free -m + nproc + git status + ipmitool -V + - name: Build with Maven + run: | + mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) + - name: Archive artifacts + run: | + mkdir -p /tmp/artifacts + tar -czf /tmp/artifacts/targets.tar.gz $(find . -name "target" -type d) tools/marvin/dist engine/schema/dist utils/conf + tar -czf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository org/apache/cloudstack + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + test: + needs: build + if: github.repository == 'apache/cloudstack' + runs-on: ubuntu-24.04 strategy: fail-fast: false matrix: @@ -215,30 +250,16 @@ jobs: smoke/test_list_service_offerings smoke/test_list_storage_pools smoke/test_list_volumes"] - steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - architecture: 'x64' - - - name: Install Build Dependencies - run: | - sudo apt-get update - sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools - + install-python: 'true' + install-apt-deps: 'true' - name: Setup IPMI Tool for CloudStack run: | # Create cloudstack-common directory if it doesn't exist @@ -256,55 +277,43 @@ jobs: /usr/share/cloudstack-common/ipmitool -C3 $@ EOF sudo chmod 755 /usr/bin/ipmitool - - name: Install Python dependencies run: | python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose - - name: Install jacoco dependencies run: | wget https://github.com/jacoco/jacoco/releases/download/v0.8.10/jacoco-0.8.10.zip unzip jacoco-0.8.10.zip -d jacoco - - - name: Env details - run: | - uname -a - whoami - javac -version - mvn -v - python3 --version - free -m - nproc - git status - ipmitool -V - - name: Setup MySQL Server run: | # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2004-Readme.md#mysql sudo apt-get install -y mysql-server sudo systemctl start mysql - sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY ''; FLUSH PRIVILEGES;" + sudo mysql -uroot -proot -e "ALTER USER 'root'@'localhost' IDENTIFIED WITH caching_sha2_password BY ''; FLUSH PRIVILEGES;" sudo systemctl restart mysql sudo mysql -uroot -e "SELECT VERSION();" - - - name: Build with Maven + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: build-artifacts + path: /tmp/artifacts/ + - name: Extract artifacts run: | - mvn -B -P developer,systemvm -Dsimulator clean install -DskipTests=true -T$(nproc) - + tar -xzf /tmp/artifacts/targets.tar.gz + mkdir -p ~/.m2/repository + tar -xzf /tmp/artifacts/m2-cloudstack.tar.gz -C ~/.m2/repository - name: Setup Simulator Prerequisites run: | sudo python3 -m pip install --upgrade netaddr mysql-connector-python python3 -m pip install --user --upgrade tools/marvin/dist/[mM]arvin-*.tar.gz mvn -q -Pdeveloper -pl developer -Ddeploydb mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator - - name: Generate jacoco-coverage.sh run: | echo "java -jar jacoco/lib/jacococli.jar report jacoco-it.exec \\" > jacoco-report.sh find . | grep "target/classes" | sed 's/\/classes\//\/classes /g' | awk '{print "--classfiles", $1, "\\"}' | sort |uniq >> jacoco-report.sh find . | grep "src/main/java" | sed 's/\/java\//\/java /g' | awk '{print "--sourcefiles", $1, "\\"}' | sort | uniq >> jacoco-report.sh echo "--xml jacoco-coverage.xml" >> jacoco-report.sh - - name: Start CloudStack Management Server with Simulator run: | export MAVEN_OPTS="-Xmx4096m -XX:MaxMetaspaceSize=800m -Djava.security.egd=file:/dev/urandom -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED" @@ -315,7 +324,6 @@ jobs: set -e echo -e "\nStarting Advanced Zone DataCenter deployment" python3 tools/marvin/marvin/deployDataCenter.py -i setup/dev/advdualzone.cfg 2>&1 || true - - name: Run Integration Tests with Simulator run: | mkdir -p integration-test-results/smoke/misc @@ -335,13 +343,12 @@ jobs: bash jacoco-report.sh mvn -Dsimulator -pl client jetty:stop 2>&1 find /tmp//MarvinLogs -type f -exec echo -e "Printing marvin logs {} :\n" \; -exec cat {} \; - - name: Integration Tests Result run: | echo -e "Simulator CI Test Results: (only failures listed)\n" python3 ./tools/marvin/xunit-reader.py integration-test-results/ - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: files: jacoco-coverage.xml fail_ci_if_error: true diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml deleted file mode 100644 index 88b10ac9178..00000000000 --- a/.github/workflows/codecov.yml +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Coverage Check - -on: [pull_request, push] - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - if: github.repository == 'apache/cloudstack' - name: codecov - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - - - name: Build CloudStack with Quality Checks - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc) - - - uses: codecov/codecov-action@v4 - with: - files: ./client/target/site/jacoco-aggregate/jacoco.xml - fail_ci_if_error: true - flags: unittests - verbose: true - name: codecov - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 31a8746b85a..a2d581dcac2 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,14 +35,16 @@ jobs: language: ["actions"] steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v3 + uses: github/codeql-action/autobuild@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: category: "Security" diff --git a/.github/workflows/daily-repo-status.lock.yml b/.github/workflows/daily-repo-status.lock.yml index 1d7e7eecd14..f8ce13c3cee 100644 --- a/.github/workflows/daily-repo-status.lock.yml +++ b/.github/workflows/daily-repo-status.lock.yml @@ -54,11 +54,11 @@ jobs: comment_repo: "" steps: - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. with: destination: /opt/gh-aw/ - name: Check workflow file - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_WORKFLOW_FILE: "daily-repo-status.lock.yml" with: @@ -96,13 +96,13 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. with: destination: /opt/gh-aw/ - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0. with: - persist-credentials: + persist-credentials: false - name: Create gh-aw temp run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir. - name: Configure Git @@ -120,7 +120,7 @@ jobs: id: checkout- if: | github.event. - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} with: @@ -132,7 +132,7 @@ jobs: await main(); - name: Generate agentic run id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # with: script: | const fs = require('fs'); @@ -469,7 +469,7 @@ jobs: } - name: Generate workflow - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # with: script: | const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); @@ -559,7 +559,7 @@ jobs: {{#runtime-import .github/workflows/daily-repo-status.md}} - name: Substitute - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. GH_AW_GITHUB_ACTOR: ${{ github.actor }} @@ -589,7 +589,7 @@ jobs: } }); - name: Interpolate variables and render - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt. with: @@ -667,7 +667,7 @@ jobs: bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" - name: Redact secrets in if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # with: script: | const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); @@ -682,7 +682,7 @@ jobs: SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload Safe if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. with: name: safe- path: ${{ env.GH_AW_SAFE_OUTPUTS }} @@ -690,7 +690,7 @@ jobs: - name: Ingest agent id: if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" @@ -704,13 +704,13 @@ jobs: await main(); - name: Upload sanitized agent if: always() && env. - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. with: name: agent- path: ${{ env.GH_AW_AGENT_OUTPUT }} if-no-files-found: - name: Upload engine output - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. with: name: path: | @@ -719,7 +719,7 @@ jobs: if-no-files-found: - name: Parse agent logs for step if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: @@ -730,7 +730,7 @@ jobs: await main(); - name: Parse MCP Gateway logs for step if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # with: script: | const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); @@ -755,7 +755,7 @@ jobs: - name: Upload agent if: always() continue-on-error: - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. with: name: agent- path: | @@ -784,12 +784,12 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. with: destination: /opt/gh-aw/ - name: Download agent output continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/safeoutputs/ @@ -800,7 +800,7 @@ jobs: echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - name: Process No-Op id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_NOOP_MAX: @@ -816,7 +816,7 @@ jobs: await main(); - name: Record Missing id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Daily Repo Status" @@ -831,7 +831,7 @@ jobs: await main(); - name: Handle Agent id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Daily Repo Status" @@ -851,7 +851,7 @@ jobs: await main(); - name: Handle No-Op id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Daily Repo Status" @@ -881,18 +881,18 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. with: destination: /opt/gh-aw/ - name: Download agent continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/threat-detection/ - name: Download agent output continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/threat-detection/ @@ -902,7 +902,7 @@ jobs: run: | echo "Agent output-types: $AGENT_OUTPUT_TYPES" - name: Setup threat - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: WORKFLOW_NAME: "Daily Repo Status" WORKFLOW_DESCRIPTION: "This workflow creates daily repo status reports. It gathers recent repository\nactivity (issues, PRs, discussions, releases, code changes) and generates\nengaging GitHub issues with productivity insights, community highlights,\nand project recommendations." @@ -955,7 +955,7 @@ jobs: XDG_CONFIG_HOME: /home/ - name: Parse threat detection id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # with: script: | const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); @@ -964,7 +964,7 @@ jobs: await main(); - name: Upload threat detection if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0. + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v6.0. with: name: threat-detection. path: /tmp/gh-aw/threat-detection/detection. @@ -993,12 +993,12 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45. + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.45. with: destination: /opt/gh-aw/ - name: Download agent output continue-on-error: - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0. + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent- path: /tmp/gh-aw/safeoutputs/ @@ -1009,7 +1009,7 @@ jobs: echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - name: Process Safe id: - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_issue\":{\"labels\":[\"report\",\"daily-status\"],\"max\":1,\"title_prefix\":\"[repo-status] \"},\"missing_data\":{},\"missing_tool\":{}}" diff --git a/.github/workflows/docker-cloudstack-simulator.yml b/.github/workflows/docker-cloudstack-simulator.yml index 8d23ac449dd..dc00968563d 100644 --- a/.github/workflows/docker-cloudstack-simulator.yml +++ b/.github/workflows/docker-cloudstack-simulator.yml @@ -35,10 +35,10 @@ concurrency: jobs: build: if: github.repository == 'apache/cloudstack' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Login to Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKERHUB_USER }} @@ -47,7 +47,9 @@ jobs: - name: Set Docker repository name run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set ACS version run: echo "ACS_VERSION=$(grep '' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV diff --git a/.github/workflows/issue-triage-agent.lock.yml b/.github/workflows/issue-triage-agent.lock.yml index 2410f7b9e45..1c3d11a800e 100644 --- a/.github/workflows/issue-triage-agent.lock.yml +++ b/.github/workflows/issue-triage-agent.lock.yml @@ -53,11 +53,11 @@ jobs: comment_repo: "" steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 with: destination: /opt/gh-aw/actions - name: Check workflow file timestamps - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_WORKFLOW_FILE: "issue-triage-agent.lock.yml" with: @@ -91,7 +91,7 @@ jobs: secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 with: destination: /opt/gh-aw/actions - name: Checkout repository @@ -113,7 +113,7 @@ jobs: echo "Git configured with standard GitHub Actions identity" - name: Generate agentic run info id: generate_aw_info - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const fs = require('fs'); @@ -167,7 +167,7 @@ jobs: run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.18.0 - name: Determine automatic lockdown mode for GitHub MCP Server id: determine-automatic-lockdown - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} @@ -459,7 +459,7 @@ jobs: } GH_AW_MCP_CONFIG_EOF - name: Generate workflow overview - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { generateWorkflowOverview } = require('/opt/gh-aw/actions/generate_workflow_overview.cjs'); @@ -552,7 +552,7 @@ jobs: {{#runtime-import .github/workflows/issue-triage-agent.md}} GH_AW_PROMPT_EOF - name: Substitute placeholders - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_GITHUB_ACTOR: ${{ github.actor }} @@ -582,7 +582,7 @@ jobs: } }); - name: Interpolate variables and render templates - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} @@ -661,7 +661,7 @@ jobs: bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" - name: Redact secrets in logs if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); @@ -676,7 +676,7 @@ jobs: SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload Safe Outputs if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: safe-output path: ${{ env.GH_AW_SAFE_OUTPUTS }} @@ -684,7 +684,7 @@ jobs: - name: Ingest agent output id: collect_output if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" @@ -698,13 +698,13 @@ jobs: await main(); - name: Upload sanitized agent output if: always() && env.GH_AW_AGENT_OUTPUT - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: agent-output path: ${{ env.GH_AW_AGENT_OUTPUT }} if-no-files-found: warn - name: Upload engine output files - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: agent_outputs path: | @@ -713,7 +713,7 @@ jobs: if-no-files-found: ignore - name: Parse agent logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ with: @@ -724,7 +724,7 @@ jobs: await main(); - name: Parse MCP Gateway logs for step summary if: always() - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); @@ -749,7 +749,7 @@ jobs: - name: Upload agent artifacts if: always() continue-on-error: true - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: agent-artifacts path: | @@ -780,12 +780,12 @@ jobs: total_count: ${{ steps.missing_tool.outputs.total_count }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 with: destination: /opt/gh-aw/actions - name: Download agent output artifact continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-output path: /tmp/gh-aw/safeoutputs/ @@ -796,7 +796,7 @@ jobs: echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - name: Process No-Op Messages id: noop - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_NOOP_MAX: 1 @@ -812,7 +812,7 @@ jobs: await main(); - name: Record Missing Tool id: missing_tool - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Issue Triage Agent" @@ -827,7 +827,7 @@ jobs: await main(); - name: Handle Agent Failure id: handle_agent_failure - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Issue Triage Agent" @@ -846,7 +846,7 @@ jobs: await main(); - name: Handle No-Op Message id: handle_noop_message - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_WORKFLOW_NAME: "Issue Triage Agent" @@ -874,18 +874,18 @@ jobs: success: ${{ steps.parse_results.outputs.success }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 with: destination: /opt/gh-aw/actions - name: Download agent artifacts continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-artifacts path: /tmp/gh-aw/threat-detection/ - name: Download agent output artifact continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-output path: /tmp/gh-aw/threat-detection/ @@ -895,7 +895,7 @@ jobs: run: | echo "Agent output-types: $AGENT_OUTPUT_TYPES" - name: Setup threat detection - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: WORKFLOW_NAME: "Issue Triage Agent" WORKFLOW_DESCRIPTION: "No description provided" @@ -948,7 +948,7 @@ jobs: XDG_CONFIG_HOME: /home/runner - name: Parse threat detection results id: parse_results - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); @@ -957,7 +957,7 @@ jobs: await main(); - name: Upload threat detection log if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: threat-detection.log path: /tmp/gh-aw/threat-detection/detection.log @@ -987,12 +987,12 @@ jobs: process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} steps: - name: Setup Scripts - uses: github/gh-aw/actions/setup@58d1d157fbac0f1204798500faefc4f7461ebe28 # v0.45.0 + uses: github/gh-aw/actions/setup@f01a9d118afa6e306f3645ca31e43f4ea8fb4d22 # v0.71.1 with: destination: /opt/gh-aw/actions - name: Download agent output artifact continue-on-error: true - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: agent-output path: /tmp/gh-aw/safeoutputs/ @@ -1003,7 +1003,7 @@ jobs: echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" - name: Process Safe Outputs id: process_safe_outputs - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":1},\"add_labels\":{\"allowed\":[\"bug\",\"feature\",\"enhancement\",\"documentation\",\"question\",\"help-wanted\",\"good-first-issue\"]},\"missing_data\":{},\"missing_tool\":{}}" diff --git a/.github/workflows/main-sonar-check.yml b/.github/workflows/main-sonar-check.yml index 7ccd6600ab9..ad58d07a989 100644 --- a/.github/workflows/main-sonar-check.yml +++ b/.github/workflows/main-sonar-check.yml @@ -15,54 +15,51 @@ # specific language governing permissions and limitations # under the License. -name: Main Branch Sonar Quality Check - +name: Sonar Quality Check (Main) +permissions: + contents: read on: push: branches: - main - -permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: if: github.repository == 'apache/cloudstack' - name: Main Sonar JaCoCo Build - runs-on: ubuntu-22.04 + name: Sonar JaCoCo Coverage + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - - - name: Run Tests with Coverage + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss + - name: Run Build and Tests with Coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack + run: mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.branch.name=${{ github.ref_name }} + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/merge-conflict-checker.yml b/.github/workflows/merge-conflict-checker.yml index a997cb94ccc..f23719f7183 100644 --- a/.github/workflows/merge-conflict-checker.yml +++ b/.github/workflows/merge-conflict-checker.yml @@ -17,28 +17,26 @@ name: "PR Merge Conflict Check" on: - push: - pull_request: - types: [opened, synchronize, reopened] + schedule: + - cron: '*/10 * * * *' + workflow_dispatch: -permissions: # added using https://github.com/step-security/secure-workflows - contents: read +permissions: {} concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: "gh-aw-${{ github.workflow }}" jobs: triage: permissions: - pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs - runs-on: ubuntu-22.04 + pull-requests: write # for eps1lon/actions-label-merge-conflict to label PRs + runs-on: ubuntu-24.04 steps: - - name: Conflict Check - uses: eps1lon/actions-label-merge-conflict@v2.0.0 - with: - repoToken: "${{ secrets.GITHUB_TOKEN }}" - dirtyLabel: "status:has-conflicts" - removeOnDirtyLabel: "status:ready-for-review" - continueOnMissingPermissions: true - commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." + - name: Conflict Check + uses: eps1lon/actions-label-merge-conflict@1df065ebe6e3310545d4f4c4e862e43bdca146f0 # v3.0.3 + with: + repoToken: "${{ secrets.GITHUB_TOKEN }}" + dirtyLabel: "status:has-conflicts" + removeOnDirtyLabel: "status:ready-for-review" + continueOnMissingPermissions: true + commentOnDirty: "This pull request has merge conflicts. Dear author, please fix the conflicts and sync your branch with the base branch." diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 895a597659d..1ea00be9112 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -29,17 +29,23 @@ concurrency: jobs: pre-commit: name: Run pre-commit - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check Out - uses: actions/checkout@v6 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + cache: 'pip' - name: Install - run: | - python -m pip install --upgrade pip - pip install pre-commit + run: pip install pre-commit - name: Set PY run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV - - uses: actions/cache@v5 + - name: Cache pre-commit environments + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.cache/pre-commit key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/rat.yml b/.github/workflows/rat.yml index 21b8e197d82..a36a956856e 100644 --- a/.github/workflows/rat.yml +++ b/.github/workflows/rat.yml @@ -16,32 +16,27 @@ # under the License. name: License Check - -on: [push, pull_request] - +on: + - push + - pull_request concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: contents: read - jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 - - name: Set up JDK 17 - uses: actions/setup-java@v5 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - java-version: '17' - distribution: 'adopt' - architecture: x64 - cache: maven + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: RAT licence checks run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss && cd nonoss && bash -x install-non-oss.sh && cd .. - rm -fr nonoss mvn -P developer,systemvm -Dsimulator -Dnoredist -pl . org.apache.rat:apache-rat-plugin:0.12:check - name: Rat Report if: always() diff --git a/.github/workflows/sonar-check.yml b/.github/workflows/sonar-check.yml index 9f5c3a194bc..49a49d00958 100644 --- a/.github/workflows/sonar-check.yml +++ b/.github/workflows/sonar-check.yml @@ -16,58 +16,52 @@ # under the License. name: Sonar Quality Check - -on: [pull_request] - permissions: - contents: read # to fetch code (actions/checkout) - pull-requests: write # for sonar to comment on pull-request - + contents: read + pull-requests: write +on: + pull_request: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build: - if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository name: Sonar JaCoCo Coverage - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: "refs/pull/${{ github.event.number }}/merge" fetch-depth: 0 - - - name: Set up JDK17 - uses: actions/setup-java@v5 + persist-credentials: false + - name: Setup Environment + uses: ./.github/actions/setup-env with: - distribution: 'temurin' - java-version: '17' - cache: 'maven' - + install-python: 'true' + install-apt-deps: 'true' - name: Cache SonarCloud packages - uses: actions/cache@v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: path: ~/.sonar/cache key: ${{ runner.os }}-sonar restore-keys: ${{ runner.os }}-sonar - - - name: Cache local Maven repository - uses: actions/cache@v5 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }} - restore-keys: | - ${{ runner.os }}-m2 - + - name: Install Non-OSS + uses: ./.github/actions/install-nonoss - name: Run Build and Tests with Coverage - id: coverage + run: mvn -B -T$(nproc) -P developer,systemvm,quality -Dsimulator -Dnoredist clean install + - name: Upload to SonarQube + if: github.repository == 'apache/cloudstack' && github.event.pull_request.head.repo.full_name == github.repository env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} PR_ID: ${{ github.event.pull_request.number }} HEADREF: ${{ github.event.pull_request.head.ref }} run: | - git clone https://github.com/shapeblue/cloudstack-nonoss.git nonoss - cd nonoss && bash -x install-non-oss.sh && cd .. - mvn -T$(nproc) -P quality -Dsimulator -Dnoredist clean install org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + mvn -B -P quality org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=apache_cloudstack -Dsonar.pullrequest.key="$PR_ID" -Dsonar.pullrequest.branch="$HEADREF" -Dsonar.pullrequest.github.repository=apache/cloudstack -Dsonar.pullrequest.provider=GitHub -Dsonar.pullrequest.github.summary_comment=true + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 + with: + files: ./client/target/site/jacoco-aggregate/jacoco.xml + fail_ci_if_error: true + flags: unittests + verbose: true + name: codecov + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 842e4497a4a..5c92f23888e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -28,7 +28,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: stale-issue-message: 'This issue is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' stale-pr-message: 'This PR is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.' @@ -41,7 +41,7 @@ jobs: days-before-pr-close: 240 exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan' exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold' - - uses: actions/stale@v10 + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: stale-issue-label: 'archive' days-before-stale: 240 diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml index 4580b6bbd5d..ff27809db8f 100644 --- a/.github/workflows/ui.yml +++ b/.github/workflows/ui.yml @@ -28,15 +28,19 @@ permissions: jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - name: Set up Node - uses: actions/setup-node@v5 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 16 + cache: 'npm' + cache-dependency-path: 'ui/package-lock.json' - name: Env details run: | @@ -55,7 +59,7 @@ jobs: npm run lint npm run test:unit - - uses: codecov/codecov-action@v4 + - uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 if: github.repository == 'apache/cloudstack' with: working-directory: ui diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 0dc5b8211e0..ba4a3874664 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -457,3 +457,18 @@ iscsi.session.cleanup.enabled=false # Instance conversion VIRT_V2V_TMPDIR env var #convert.instance.env.virtv2v.tmpdir= + +# Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. +# incremental.snapshot.retry.rebase.wait=60 + +# Path to the VDDK library directory for VMware to KVM conversion via VDDK, +# passed to virt-v2v as -io vddk-libdir= +#vddk.lib.dir= + +# Ordered VDDK transport preference for VMware to KVM conversion via VDDK, passed as +# -io vddk-transports= to virt-v2v. Example: nbd:nbdssl +#vddk.transports= + +# Optional vCenter SHA1 thumbprint for VMware to KVM conversion via VDDK, passed as +# -io vddk-thumbprint=. If unset, CloudStack computes it on the KVM host via openssl. +#vddk.thumbprint= diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 3364f9708cf..e69a7efdc9c 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -808,6 +808,30 @@ public class AgentProperties{ */ public static final Property CONVERT_ENV_VIRTV2V_TMPDIR = new Property<>("convert.instance.env.virtv2v.tmpdir", null, String.class); + /** + * Path to the VDDK library directory on the KVM conversion host, used when converting VMs from VMware to KVM via VDDK. + * This directory is passed to virt-v2v as -io vddk-libdir=<path>. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_LIB_DIR = new Property<>("vddk.lib.dir", null, String.class); + + /** + * Ordered list of VDDK transports for virt-v2v, passed as -io vddk-transports=<value>. + * Example: nbd:nbdssl. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_TRANSPORTS = new Property<>("vddk.transports", null, String.class); + + /** + * vCenter TLS certificate thumbprint used by virt-v2v VDDK mode, passed as -io vddk-thumbprint=<value>. + * If unset, the KVM host computes it at runtime from the vCenter endpoint. + * Data type: String.
+ * Default value: null + */ + public static final Property VDDK_THUMBPRINT = new Property<>("vddk.thumbprint", null, String.class); + /** * BGP controll CIDR * Data type: String.
@@ -885,6 +909,11 @@ public class AgentProperties{ */ public static final Property CREATE_FULL_CLONE = new Property<>("create.full.clone", false); + /** + * Time, in seconds, to wait before retrying to rebase during the incremental snapshot process. + * */ + public static final Property INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT = new Property<>("incremental.snapshot.retry.rebase.wait", 60); + public static class Property { private String name; diff --git a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java index f7e4bfea80f..fd8237998a7 100644 --- a/api/src/main/java/com/cloud/agent/api/to/BucketTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/BucketTO.java @@ -26,10 +26,13 @@ public final class BucketTO { private String secretKey; + private long accountId; + public BucketTO(Bucket bucket) { this.name = bucket.getName(); this.accessKey = bucket.getAccessKey(); this.secretKey = bucket.getSecretKey(); + this.accountId = bucket.getAccountId(); } public BucketTO(String name) { @@ -47,4 +50,8 @@ public final class BucketTO { public String getSecretKey() { return this.secretKey; } + + public long getAccountId() { + return this.accountId; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java index 18737c584b3..7daeb964917 100644 --- a/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java @@ -36,13 +36,17 @@ public class RemoteInstanceTO implements Serializable { private String vcenterPassword; private String vcenterHost; private String datacenterName; + private String clusterName; + private String hostName; public RemoteInstanceTO() { } - public RemoteInstanceTO(String instanceName) { + public RemoteInstanceTO(String instanceName, String clusterName, String hostName) { this.hypervisorType = Hypervisor.HypervisorType.VMware; this.instanceName = instanceName; + this.clusterName = clusterName; + this.hostName = hostName; } public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) { @@ -55,6 +59,12 @@ public class RemoteInstanceTO implements Serializable { this.datacenterName = datacenterName; } + public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName, String clusterName, String hostName) { + this(instanceName, instancePath, vcenterHost, vcenterUsername, vcenterPassword, datacenterName); + this.clusterName = clusterName; + this.hostName = hostName; + } + public Hypervisor.HypervisorType getHypervisorType() { return this.hypervisorType; } @@ -82,4 +92,12 @@ public class RemoteInstanceTO implements Serializable { public String getDatacenterName() { return datacenterName; } + + public String getClusterName() { + return clusterName; + } + + public String getHostName() { + return hostName; + } } diff --git a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java index e26cc1e9f02..9af6c731fd2 100644 --- a/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java +++ b/api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java @@ -51,6 +51,7 @@ public class VirtualMachineTO { private long minRam; private long maxRam; + private long requestedRam; private String hostName; private String arch; private String os; @@ -207,15 +208,20 @@ public class VirtualMachineTO { return minRam; } - public void setRam(long minRam, long maxRam) { + public void setRam(long minRam, long maxRam, long requestedRam) { this.minRam = minRam; this.maxRam = maxRam; + this.requestedRam = requestedRam; } public long getMaxRam() { return maxRam; } + public long getRequestedRam() { + return requestedRam; + } + public String getHostName() { return hostName; } diff --git a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java index 604720aaa29..5d028d31d5b 100644 --- a/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java +++ b/api/src/main/java/com/cloud/agent/manager/allocator/HostAllocator.java @@ -22,19 +22,11 @@ import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.Host; import com.cloud.host.Host.Type; -import com.cloud.offering.ServiceOffering; import com.cloud.utils.component.Adapter; -import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineProfile; public interface HostAllocator extends Adapter { - /** - * @param UserVm vm - * @param ServiceOffering offering - **/ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - /** * Determines which physical hosts are suitable to * allocate the guest virtual machines on @@ -49,31 +41,6 @@ public interface HostAllocator extends Adapter { public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo); - /** - * Determines which physical hosts are suitable to allocate the guest - * virtual machines on - * - * Allocators must set any other hosts not considered for allocation in the - * ExcludeList avoid. Thus the avoid set and the list of hosts suitable, - * together must cover the entire host set in the cluster. - * - * @param VirtualMachineProfile - * vmProfile - * @param DeploymentPlan - * plan - * @param GuestType - * type - * @param ExcludeList - * avoid - * @param int returnUpTo (use -1 to return all possible hosts) - * @param boolean considerReservedCapacity (default should be true, set to - * false if host capacity calculation should not look at reserved - * capacity) - * @return List List of hosts that are suitable for VM allocation - **/ - - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity); - /** * Determines which physical hosts are suitable to allocate the guest * virtual machines on diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 8f7e773070f..22d796d4a77 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing; + random, firstfit, userdispersing, firstfitleastconsumed; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/com/cloud/ha/Investigator.java b/api/src/main/java/com/cloud/ha/Investigator.java index 88d802a1ce4..00371d395f5 100644 --- a/api/src/main/java/com/cloud/ha/Investigator.java +++ b/api/src/main/java/com/cloud/ha/Investigator.java @@ -26,17 +26,19 @@ public interface Investigator extends Adapter { * Returns if the vm is still alive. * * @param vm to work on. + * @return true if vm is alive, otherwise false */ - public boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM; + boolean isVmAlive(VirtualMachine vm, Host host) throws UnknownVM; - public Status isAgentAlive(Host agent); + /** + * Returns the agent status of the host. + * + * @param host + * @return status of the host agent + */ + Status getHostAgentStatus(Host host); class UnknownVM extends Exception { - - /** - * - */ private static final long serialVersionUID = 1L; - }; } diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index 9c011bac319..b5234820151 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -57,6 +57,9 @@ public interface Host extends StateObject, Identity, Partition, HAResour String HOST_UEFI_ENABLE = "host.uefi.enable"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; + String HOST_VDDK_SUPPORT = "host.vddk.support"; + String HOST_VDDK_LIB_DIR = "vddk.lib.dir"; + String HOST_VDDK_VERSION = "host.vddk.version"; String HOST_OVFTOOL_VERSION = "host.ovftool.version"; String HOST_VIRTV2V_VERSION = "host.virtv2v.version"; String HOST_SSH_PORT = "host.ssh.port"; diff --git a/api/src/main/java/com/cloud/network/Network.java b/api/src/main/java/com/cloud/network/Network.java index 0846306f70f..e41eb880ffd 100644 --- a/api/src/main/java/com/cloud/network/Network.java +++ b/api/src/main/java/com/cloud/network/Network.java @@ -510,4 +510,6 @@ public interface Network extends ControlledEntity, StateObject, I Integer getPrivateMtu(); Integer getNetworkCidrSize(); + + boolean getKeepMacAddressOnPublicNic(); } diff --git a/api/src/main/java/com/cloud/network/NetworkProfile.java b/api/src/main/java/com/cloud/network/NetworkProfile.java index 2e8efb48930..d690344a0e3 100644 --- a/api/src/main/java/com/cloud/network/NetworkProfile.java +++ b/api/src/main/java/com/cloud/network/NetworkProfile.java @@ -385,6 +385,11 @@ public class NetworkProfile implements Network { return networkCidrSize; } + @Override + public boolean getKeepMacAddressOnPublicNic() { + return true; + } + @Override public String toString() { return String.format("NetworkProfile %s", diff --git a/api/src/main/java/com/cloud/network/vpc/Vpc.java b/api/src/main/java/com/cloud/network/vpc/Vpc.java index b94089d2d43..a0686e2bf7d 100644 --- a/api/src/main/java/com/cloud/network/vpc/Vpc.java +++ b/api/src/main/java/com/cloud/network/vpc/Vpc.java @@ -107,4 +107,6 @@ public interface Vpc extends ControlledEntity, Identity, InternalIdentity { String getIp6Dns2(); boolean useRouterIpAsResolver(); + + boolean getKeepMacAddressOnPublicNic(); } diff --git a/api/src/main/java/com/cloud/network/vpc/VpcService.java b/api/src/main/java/com/cloud/network/vpc/VpcService.java index c1546609d2b..3d0ba43263f 100644 --- a/api/src/main/java/com/cloud/network/vpc/VpcService.java +++ b/api/src/main/java/com/cloud/network/vpc/VpcService.java @@ -58,7 +58,7 @@ public interface VpcService { */ Vpc createVpc(long zoneId, long vpcOffId, long vpcOwnerId, String vpcName, String displayText, String cidr, String networkDomain, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Boolean displayVpc, Integer publicMtu, Integer cidrSize, - Long asNumber, List bgpPeerIds, Boolean useVrIpResolver) throws ResourceAllocationException; + Long asNumber, List bgpPeerIds, Boolean useVrIpResolver, boolean keepMacAddressOnPublicNic) throws ResourceAllocationException; /** * Persists VPC record in the database @@ -104,7 +104,7 @@ public interface VpcService { * @throws ResourceUnavailableException if during restart some resources may not be available * @throws InsufficientCapacityException if for instance no address space, compute or storage is sufficiently available */ - Vpc updateVpc(long vpcId, String vpcName, String displayText, String customId, Boolean displayVpc, Integer mtu, String sourceNatIp) throws ResourceUnavailableException, InsufficientCapacityException; + Vpc updateVpc(long vpcId, String vpcName, String displayText, String customId, Boolean displayVpc, Integer mtu, String sourceNatIp, Boolean keepMacAddressOnPublicNic) throws ResourceUnavailableException, InsufficientCapacityException; /** * Lists VPC(s) based on the parameters passed to the API call diff --git a/api/src/main/java/com/cloud/projects/ProjectService.java b/api/src/main/java/com/cloud/projects/ProjectService.java index 5080cb5a781..d11e9ae0446 100644 --- a/api/src/main/java/com/cloud/projects/ProjectService.java +++ b/api/src/main/java/com/cloud/projects/ProjectService.java @@ -82,7 +82,7 @@ public interface ProjectService { Project updateProject(long id, String name, String displayText, String newOwnerName, Long userId, Role newRole) throws ResourceAllocationException; - boolean addAccountToProject(long projectId, String accountName, String email, Long projectRoleId, Role projectRoleType); + boolean addAccountToProject(long projectId, String accountName, String email, Long projectRoleId, Role projectRoleType) throws ResourceAllocationException; boolean deleteAccountFromProject(long projectId, String accountName); @@ -100,6 +100,6 @@ public interface ProjectService { Project findByProjectAccountIdIncludingRemoved(long projectAccountId); - boolean addUserToProject(Long projectId, String username, String email, Long projectRoleId, Role projectRole); + boolean addUserToProject(Long projectId, String username, String email, Long projectRoleId, Role projectRole) throws ResourceAllocationException; } diff --git a/api/src/main/java/com/cloud/server/ManagementService.java b/api/src/main/java/com/cloud/server/ManagementService.java index 51737546ffa..bcca229c06b 100644 --- a/api/src/main/java/com/cloud/server/ManagementService.java +++ b/api/src/main/java/com/cloud/server/ManagementService.java @@ -71,7 +71,6 @@ import org.apache.cloudstack.api.command.user.vm.GetVMPasswordCmd; import org.apache.cloudstack.api.command.user.vmgroup.UpdateVMGroupCmd; import org.apache.cloudstack.config.Configuration; import org.apache.cloudstack.config.ConfigurationGroup; -import org.apache.cloudstack.framework.config.ConfigKey; import com.cloud.alert.Alert; import com.cloud.capacity.Capacity; @@ -108,14 +107,6 @@ import com.cloud.vm.VirtualMachineProfile; public interface ManagementService { static final String Name = "management-server"; - ConfigKey JsInterpretationEnabled = new ConfigKey<>("Hidden" - , Boolean.class - , "js.interpretation.enabled" - , "false" - , "Enable/Disable all JavaScript interpretation related functionalities to create or update Javascript rules." - , false - , ConfigKey.Scope.Global); - /** * returns the a map of the names/values in the configuration table * @@ -534,6 +525,4 @@ public interface ManagementService { boolean removeManagementServer(RemoveManagementServerCmd cmd); - void checkJsInterpretationAllowedIfNeededForParameterValue(String paramName, boolean paramValue); - } diff --git a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java index db702a61f2b..7d5b2d7c57d 100644 --- a/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java +++ b/api/src/main/java/com/cloud/storage/VMTemplateStorageResourceAssoc.java @@ -23,9 +23,10 @@ import org.apache.cloudstack.api.InternalIdentity; public interface VMTemplateStorageResourceAssoc extends InternalIdentity { public static enum Status { - UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED + UNKNOWN, DOWNLOAD_ERROR, NOT_DOWNLOADED, DOWNLOAD_IN_PROGRESS, DOWNLOADED, ABANDONED, LIMIT_REACHED, UPLOADED, NOT_UPLOADED, UPLOAD_ERROR, UPLOAD_IN_PROGRESS, CREATING, CREATED, BYPASSED } + List ERROR_DOWNLOAD_STATES = List.of(Status.DOWNLOAD_ERROR, Status.ABANDONED, Status.LIMIT_REACHED, Status.UNKNOWN); List PENDING_DOWNLOAD_STATES = List.of(Status.NOT_DOWNLOADED, Status.DOWNLOAD_IN_PROGRESS); String getInstallPath(); diff --git a/api/src/main/java/com/cloud/user/ResourceLimitService.java b/api/src/main/java/com/cloud/user/ResourceLimitService.java index 738e593582b..9c493fb383c 100644 --- a/api/src/main/java/com/cloud/user/ResourceLimitService.java +++ b/api/src/main/java/com/cloud/user/ResourceLimitService.java @@ -30,6 +30,7 @@ import com.cloud.exception.ResourceAllocationException; import com.cloud.offering.DiskOffering; import com.cloud.offering.ServiceOffering; import com.cloud.template.VirtualMachineTemplate; +import org.apache.cloudstack.resourcelimit.Reserver; public interface ResourceLimitService { @@ -191,6 +192,7 @@ public interface ResourceLimitService { */ public void checkResourceLimit(Account account, ResourceCount.ResourceType type, long... count) throws ResourceAllocationException; public void checkResourceLimitWithTag(Account account, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; + public void checkResourceLimitWithTag(Account account, Long domainId, boolean considerSystemAccount, ResourceCount.ResourceType type, String tag, long... count) throws ResourceAllocationException; /** * Gets the count of resources for a resource type and account @@ -251,12 +253,12 @@ public interface ResourceLimitService { List getResourceLimitStorageTags(DiskOffering diskOffering); void updateTaggedResourceLimitsAndCountsForAccounts(List responses, String tag); void updateTaggedResourceLimitsAndCountsForDomains(List responses, String tag); - void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; - + void checkVolumeResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List reservations) throws ResourceAllocationException; + List getResourceLimitStorageTagsForResourceCountOperation(Boolean display, DiskOffering diskOffering); void checkVolumeResourceLimitForDiskOfferingChange(Account owner, Boolean display, Long currentSize, Long newSize, - DiskOffering currentOffering, DiskOffering newOffering) throws ResourceAllocationException; + DiskOffering currentOffering, DiskOffering newOffering, List reservations) throws ResourceAllocationException; - void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering) throws ResourceAllocationException; + void checkPrimaryStorageResourceLimit(Account owner, Boolean display, Long size, DiskOffering diskOffering, List reservations) throws ResourceAllocationException; void incrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumeResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); @@ -273,25 +275,23 @@ public interface ResourceLimitService { void incrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); void decrementVolumePrimaryStorageResourceCount(long accountId, Boolean display, Long size, DiskOffering diskOffering); - void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + void checkVmResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, List reservations) throws ResourceAllocationException; void incrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); void decrementVmResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template); void checkVmResourceLimitsForServiceOfferingChange(Account owner, Boolean display, Long currentCpu, Long newCpu, - Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template) throws ResourceAllocationException; + Long currentMemory, Long newMemory, ServiceOffering currentOffering, ServiceOffering newOffering, VirtualMachineTemplate template, List reservations) throws ResourceAllocationException; void checkVmResourceLimitsForTemplateChange(Account owner, Boolean display, ServiceOffering offering, - VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate) throws ResourceAllocationException; + VirtualMachineTemplate currentTemplate, VirtualMachineTemplate newTemplate, List reservations) throws ResourceAllocationException; - void checkVmCpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu) throws ResourceAllocationException; void incrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); void decrementVmCpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long cpu); - void checkVmMemoryResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory) throws ResourceAllocationException; void incrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); void decrementVmMemoryResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long memory); - void checkVmGpuResourceLimit(Account owner, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu) throws ResourceAllocationException; void incrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu); void decrementVmGpuResourceCount(long accountId, Boolean display, ServiceOffering serviceOffering, VirtualMachineTemplate template, Long gpu); + long recalculateDomainResourceCount(final long domainId, final ResourceType type, String tag); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java index 4d33ba859a5..e2ebb242cbf 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java @@ -127,8 +127,8 @@ public enum ApiCommandResourceType { } public static ApiCommandResourceType fromString(String value) { - if (StringUtils.isNotEmpty(value) && EnumUtils.isValidEnum(ApiCommandResourceType.class, value)) { - return valueOf(value); + if (StringUtils.isNotBlank(value) && EnumUtils.isValidEnumIgnoreCase(ApiCommandResourceType.class, value)) { + return EnumUtils.getEnumIgnoreCase(ApiCommandResourceType.class, value); } return null; } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 05c6098bc72..694830ea2f3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -69,6 +69,8 @@ public class ApiConstants { public static final String BACKUP_VM_OFFERING_REMOVED = "vmbackupofferingremoved"; public static final String IS_BACKUP_VM_EXPUNGED = "isbackupvmexpunged"; public static final String BACKUP_TOTAL = "backuptotal"; + public static final String BALANCE = "balance"; + public static final String BALANCES = "balances"; public static final String BASE64_IMAGE = "base64image"; public static final String BGP_PEERS = "bgppeers"; public static final String BGP_PEER_IDS = "bgppeerids"; @@ -157,6 +159,7 @@ public class ApiConstants { public static final String CUSTOM_ID = "customid"; public static final String CUSTOM_ACTION_ID = "customactionid"; public static final String CUSTOM_JOB_ID = "customjobid"; + public static final String CURRENCY = "currency"; public static final String CURRENT_START_IP = "currentstartip"; public static final String CURRENT_END_IP = "currentendip"; public static final String ENCRYPT = "encrypt"; @@ -170,6 +173,7 @@ public class ApiConstants { public static final String DATACENTER_NAME = "datacentername"; public static final String DATADISKS_DETAILS = "datadisksdetails"; public static final String DATADISK_OFFERING_LIST = "datadiskofferinglist"; + public static final String DATE = "date"; public static final String DEFAULT_VALUE = "defaultvalue"; public static final String DELETE_PROTECTION = "deleteprotection"; public static final String DESCRIPTION = "description"; @@ -509,6 +513,7 @@ public class ApiConstants { public static final String REPAIR = "repair"; public static final String REPETITION_ALLOWED = "repetitionallowed"; public static final String REQUIRES_HVM = "requireshvm"; + public static final String RESERVED_RESOURCE_DETAILS = "reservedresourcedetails"; public static final String RESOURCES = "resources"; public static final String RESOURCE_COUNT = "resourcecount"; public static final String RESOURCE_NAME = "resourcename"; @@ -525,7 +530,6 @@ public class ApiConstants { public static final String SCHEDULE = "schedule"; public static final String SCHEDULE_ID = "scheduleid"; public static final String SCOPE = "scope"; - public static final String USER_SECRET_KEY = "usersecretkey"; public static final String SEARCH_BASE = "searchbase"; public static final String SECONDARY_IP = "secondaryip"; public static final String SECURITY_GROUP_IDS = "securitygroupids"; @@ -541,6 +545,7 @@ public class ApiConstants { public static final String SESSIONKEY = "sessionkey"; public static final String SHOW_CAPACITIES = "showcapacities"; public static final String SHOW_REMOVED = "showremoved"; + public static final String SHOW_RESOURCES = "showresources"; public static final String SHOW_RESOURCE_ICON = "showicon"; public static final String SHOW_INACTIVE = "showinactive"; public static final String SHOW_UNIQUE = "showunique"; @@ -606,9 +611,11 @@ public class ApiConstants { public static final String TENANT_NAME = "tenantname"; public static final String TOTAL = "total"; public static final String TOTAL_SUBNETS = "totalsubnets"; + public static final String TOTAL_QUOTA = "totalquota"; public static final String TYPE = "type"; public static final String TRUST_STORE = "truststore"; public static final String TRUST_STORE_PASSWORD = "truststorepass"; + public static final String UNIT = "unit"; public static final String URL = "url"; public static final String USAGE_INTERFACE = "usageinterface"; public static final String USED = "used"; @@ -629,6 +636,8 @@ public class ApiConstants { public static final String USERNAME = "username"; public static final String USER_CONFIGURABLE = "userconfigurable"; public static final String USER_SECURITY_GROUP_LIST = "usersecuritygrouplist"; + public static final String USER_SECRET_KEY = "usersecretkey"; + public static final String USE_VDDK = "usevddk"; public static final String USE_VIRTUAL_NETWORK = "usevirtualnetwork"; public static final String USE_VIRTUAL_ROUTER_IP_RESOLVER = "userouteripresolver"; public static final String UPDATE_IN_SEQUENCE = "updateinsequence"; @@ -1298,6 +1307,8 @@ public class ApiConstants { public static final String OBJECT_LOCKING = "objectlocking"; public static final String ENCRYPTION = "encryption"; public static final String QUOTA = "quota"; + public static final String QUOTA_CONSUMED = "quotaconsumed"; + public static final String QUOTA_USAGE = "quotausage"; public static final String ACCESS_KEY = "accesskey"; public static final String SOURCE_NAT_IP = "sourcenatipaddress"; @@ -1348,6 +1359,13 @@ public class ApiConstants { public static final String OBJECT_STORAGE_LIMIT = "objectstoragelimit"; public static final String OBJECT_STORAGE_TOTAL = "objectstoragetotal"; + public static final String KEEP_MAC_ADDRESS_ON_PUBLIC_NIC = "keepmacaddressonpublicnic"; + + public static final String PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC = + "Indicates whether to use the same MAC address for the public NIC of VRs on the same network. If \"true\", when creating redundant routers or recreating" + + " a VR, CloudStack will use the same MAC address for the public NIC of all VRs. Otherwise, if \"false\", new public NICs will always have " + + " a new MAC address."; + public static final String PARAMETER_DESCRIPTION_ACTIVATION_RULE = "Quota tariff's activation rule. It can receive a JS script that results in either " + "a boolean or a numeric value: if it results in a boolean value, the tariff value will be applied according to the result; if it results in a numeric value, the " + "numeric value will be applied; if the result is neither a boolean nor a numeric value, the tariff will not be applied. If the rule is not informed, the tariff " + diff --git a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java index e495cf28413..00b1bc310d5 100644 --- a/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/BaseCmd.java @@ -27,7 +27,6 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.regex.Pattern; import javax.inject.Inject; @@ -504,12 +503,6 @@ public abstract class BaseCmd { } public String getResourceUuid(String parameterName) { - UUID resourceUuid = CallContext.current().getApiResourceUuid(parameterName); - - if (resourceUuid != null) { - return resourceUuid.toString(); - } - - return null; + return CallContext.current().getApiResourceUuid(parameterName); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java index 6deaea22ac6..d333a74fdb3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/ca/ProvisionCertificateCmd.java @@ -63,6 +63,12 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd { description = "Name of the CA service provider, otherwise the default configured provider plugin will be used") private String provider; + @Parameter(name = ApiConstants.FORCED, type = CommandType.BOOLEAN, + description = "When true, uses SSH to re-provision the agent's certificate, bypassing the NIO agent connection. " + + "Use this when agents are disconnected due to a CA change. Supported for KVM hosts and SystemVMs. Default is false", + since = "4.23.0") + private Boolean forced; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -79,6 +85,10 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd { return provider; } + public boolean isForced() { + return forced != null && forced; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// @@ -90,7 +100,7 @@ public class ProvisionCertificateCmd extends BaseAsyncCmd { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Unable to find host by ID: " + getHostId()); } - boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider()); + boolean result = caManager.provisionCertificate(host, getReconnect(), getProvider(), isForced()); SuccessResponse response = new SuccessResponse(getCommandName()); response.setSuccess(result); setResponseObject(response); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java index 6804e4355ca..cb427e65949 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/ha/ConfigureHAForHostCmd.java @@ -87,6 +87,7 @@ public final class ConfigureHAForHostCmd extends BaseAsyncCmd { final HostHAResponse response = new HostHAResponse(); response.setId(resourceUuid); response.setProvider(getHaProvider().toLowerCase()); + response.setStatus(result); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java index abca619f82a..4d6ef741961 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/FindHostsForMigrationCmd.java @@ -78,7 +78,7 @@ public class FindHostsForMigrationCmd extends BaseListCmd { for (Host host : result.first()) { HostForMigrationResponse hostResponse = _responseGenerator.createHostForMigrationResponse(host); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java index e202dfad77b..8f5e6c784d6 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java @@ -252,7 +252,7 @@ public class ListHostsCmd extends BaseListCmd { for (Host host : result.first()) { HostResponse hostResponse = _responseGenerator.createHostResponse(host, getDetails()); Boolean suitableForMigration = false; - if (hostsWithCapacity.contains(host)) { + if (hostsWithCapacity != null && hostsWithCapacity.contains(host)) { suitableForMigration = true; } hostResponse.setSuitableForMigration(suitableForMigration); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java index 3427cef3366..a3dcf08c3a3 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/user/GetUserCmd.java @@ -22,7 +22,7 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.UserResponse; - +import org.apache.cloudstack.api.ApiArgValidator; import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.UserAccount; @@ -35,7 +35,7 @@ public class GetUserCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user") + @Parameter(name = ApiConstants.USER_API_KEY, type = CommandType.STRING, required = true, description = "API key of the user", validations = {ApiArgValidator.NotNullOrEmpty}) private String apiKey; ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index 50ccfbd69c5..db7dcc3fb44 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -179,6 +179,14 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { description = "(only for importing VMs from VMware to KVM) optional - the ID of the guest OS for the imported VM.") private Long guestOsId; + @Parameter(name = ApiConstants.USE_VDDK, + type = CommandType.BOOLEAN, + since = "4.22.1", + description = "(only for importing VMs from VMware to KVM) optional - if true, uses VDDK on the KVM conversion host for converting the VM. " + + "This parameter is mutually exclusive with " + ApiConstants.FORCE_MS_TO_IMPORT_VM_FILES + ".") + private Boolean useVddk; + + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -255,6 +263,10 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { return storagePoolId; } + public boolean getUseVddk() { + return BooleanUtils.toBooleanDefaultIfNull(useVddk, true); + } + public String getTmpPath() { return tmpPath; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java index 85e7b0af319..63a0a6ca51e 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddAccountToProjectCmd.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.api.command.user.account; import java.util.List; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.BaseCmd; @@ -106,7 +107,7 @@ public class AddAccountToProjectCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// @Override - public void execute() { + public void execute() throws ResourceAllocationException { if (accountName == null && email == null) { throw new InvalidParameterValueException("Either accountName or email is required"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java index 69832d4dfdc..683522039b1 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/account/AddUserToProjectCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.account; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiArgValidator; @@ -111,7 +112,7 @@ public class AddUserToProjectCmd extends BaseAsyncCmd { ///////////////////////////////////////////////////// @Override - public void execute() { + public void execute() throws ResourceAllocationException { validateInput(); boolean result = _projectService.addUserToProject(getProjectId(), getUsername(), getEmail(), getProjectRoleId(), getRoleType()); if (result) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java index 28cd642e1a4..29b1e740b67 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/AssignVirtualMachineToBackupOfferingCmd.java @@ -21,6 +21,7 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -102,6 +103,16 @@ public class AssignVirtualMachineToBackupOfferingCmd extends BaseAsyncCmd { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_ASSIGN; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java index ca60ea674fe..8a4053aa15d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java @@ -123,7 +123,12 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd { @Override public ApiCommandResourceType getApiResourceType() { - return ApiCommandResourceType.Backup; + return ApiCommandResourceType.VirtualMachine; + } + + @Override + public Long getApiResourceId() { + return vmId; } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java index 67ad7c71503..f6e17a2b390 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java @@ -21,6 +21,7 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseCmd; @@ -81,7 +82,7 @@ public class CreateBackupScheduleCmd extends BaseCmd { @Parameter(name = ApiConstants.QUIESCE_VM, type = CommandType.BOOLEAN, required = false, - description = "Quiesce the instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + + description = "Quiesce the Instance before checkpointing the disks for backup. Applicable only to NAS backup provider. " + "The filesystem is frozen before the backup starts and thawed immediately after. " + "Requires the instance to have the QEMU Guest Agent installed and running.", since = "4.21.0") @@ -139,4 +140,14 @@ public class CreateBackupScheduleCmd extends BaseCmd { public long getEntityOwnerId() { return CallContext.current().getCallingAccount().getId(); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java index dcf9f15b4dc..62c2f03d880 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RemoveVirtualMachineFromBackupOfferingCmd.java @@ -21,6 +21,7 @@ import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -99,6 +100,16 @@ public class RemoveVirtualMachineFromBackupOfferingCmd extends BaseAsyncCmd { return CallContext.current().getCallingAccount().getId(); } + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } + @Override public String getEventType() { return EventTypes.EVENT_VM_BACKUP_OFFERING_REMOVE; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java index 703a1b2e880..c15e6f8de68 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/backup/RestoreVolumeFromBackupAndAttachToVMCmd.java @@ -20,7 +20,9 @@ package org.apache.cloudstack.api.command.user.backup; import javax.inject.Inject; import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.BaseAsyncCmd; @@ -53,6 +55,7 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, @@ -60,12 +63,14 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { description = "ID of the Instance backup") private Long backupId; + @ACL @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.STRING, required = true, description = "ID of the volume backed up") private String volumeUuid; + @ACL @Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, @@ -123,4 +128,14 @@ public class RestoreVolumeFromBackupAndAttachToVMCmd extends BaseAsyncCmd { public String getEventDescription() { return "Restoring volume "+ volumeUuid + " from backup " + getResourceUuid(ApiConstants.BACKUP_ID) + " and attaching it to Instance " + getResourceUuid(ApiConstants.VIRTUAL_MACHINE_ID); } + + @Override + public Long getApiResourceId() { + return vmId; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.VirtualMachine; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java index 4a9a0569c3b..abbb1760f9d 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/bucket/DeleteBucketCmd.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.api.command.user.bucket; import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.storage.object.Bucket; import com.cloud.user.Account; @@ -82,7 +83,7 @@ public class DeleteBucketCmd extends BaseCmd { } @Override - public void execute() throws ConcurrentOperationException { + public void execute() throws ConcurrentOperationException, ResourceAllocationException { CallContext.current().setEventDetails("Bucket ID: " + getResourceUuid(ApiConstants.ID)); boolean result = _bucketService.deleteBucket(id, CallContext.current().getCallingAccount()); SuccessResponse response = new SuccessResponse(getCommandName()); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java index b55d1b234f1..2c840183113 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/ListAsyncJobsCmd.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.api.command.user.job; import java.util.Date; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListAccountResourcesCmd; import org.apache.cloudstack.api.Parameter; @@ -40,6 +41,12 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "The id of the management server", since="4.19") private Long managementServerId; + @Parameter(name = ApiConstants.RESOURCE_ID, validations = {ApiArgValidator.UuidString}, type = CommandType.STRING, description = "the ID of the resource associated with the job", since="4.22.1") + private String resourceId; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") + private String resourceType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -52,6 +59,14 @@ public class ListAsyncJobsCmd extends BaseListAccountResourcesCmd { return managementServerId; } + public String getResourceId() { + return resourceId; + } + + public String getResourceType() { + return resourceType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java index 93a44375721..5c3b0084574 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/job/QueryAsyncJobResultCmd.java @@ -16,8 +16,8 @@ // under the License. package org.apache.cloudstack.api.command.user.job; - import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiArgValidator; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; @@ -34,9 +34,15 @@ public class QueryAsyncJobResultCmd extends BaseCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// - @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, required = true, description = "The ID of the asynchronous job") + @Parameter(name = ApiConstants.JOB_ID, type = CommandType.UUID, entityType = AsyncJobResponse.class, description = "The ID of the asynchronous job") private Long id; + @Parameter(name = ApiConstants.RESOURCE_ID, validations = {ApiArgValidator.UuidString}, type = CommandType.STRING, description = "the ID of the resource associated with the job", since="4.22.1") + private String resourceId; + + @Parameter(name = ApiConstants.RESOURCE_TYPE, type = CommandType.STRING, description = "the type of the resource associated with the job", since="4.22.1") + private String resourceType; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -45,6 +51,14 @@ public class QueryAsyncJobResultCmd extends BaseCmd { return id; } + public String getResourceId() { + return resourceId; + } + + public String getResourceType() { + return resourceType; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java index cbf6df081b3..ee5b8568e83 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/CreateNetworkCmd.java @@ -199,6 +199,11 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { @Parameter(name=ApiConstants.AS_NUMBER, type=CommandType.LONG, since = "4.20.0", description="the AS Number of the network") private Long asNumber; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -286,6 +291,10 @@ public class CreateNetworkCmd extends BaseCmd implements UserCmd { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + @Override public boolean isDisplay() { if(displayNetwork == null) diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java index 2e638f1e2f7..7b6841d6097 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/network/UpdateNetworkCmd.java @@ -105,6 +105,11 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd implements UserCmd { @Parameter(name = ApiConstants.SOURCE_NAT_IP, type = CommandType.STRING, description = "IPV4 address to be assigned to the public interface of the network router. This address must already be acquired for this network", since = "4.19") private String sourceNatIP; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -186,6 +191,10 @@ public class UpdateNetworkCmd extends BaseAsyncCustomIdCmd implements UserCmd { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index e17ba9c2d70..a719062e1bc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -51,6 +51,7 @@ public class CreateVMFromBackupCmd extends BaseDeployVMCmd { //////////////// API parameters ///////////////////// ///////////////////////////////////////////////////// + @ACL @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java index 5bcf3a14117..78de0564848 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/volume/CreateVolumeCmd.java @@ -32,6 +32,7 @@ import org.apache.cloudstack.api.response.DiskOfferingResponse; import org.apache.cloudstack.api.response.DomainResponse; import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.UserVmResponse; import org.apache.cloudstack.api.response.VolumeResponse; import org.apache.cloudstack.api.response.ZoneResponse; @@ -109,6 +110,13 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC description = "The ID of the Instance; to be used with snapshot Id, Instance to which the volume gets attached after creation") private Long virtualMachineId; + @Parameter(name = ApiConstants.STORAGE_ID, + type = CommandType.UUID, + entityType = StoragePoolResponse.class, + description = "Storage pool ID to create the volume in. Cannot be used with the snapshotid parameter.", + authorized = {RoleType.Admin}) + private Long storageId; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -153,6 +161,13 @@ public class CreateVolumeCmd extends BaseAsyncCreateCustomIdCmd implements UserC return projectId; } + public Long getStorageId() { + if (snapshotId != null && storageId != null) { + throw new IllegalArgumentException("StorageId parameter cannot be specified with the SnapshotId parameter."); + } + return storageId; + } + public Boolean getDisplayVolume() { return displayVolume; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java index 2adbbd66408..86309d7f28a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/CreateVPCCmd.java @@ -130,6 +130,11 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { description="(optional) for NSX based VPCs: when set to true, use the VR IP as nameserver, otherwise use DNS1 and DNS2") private Boolean useVrIpResolver; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private boolean keepMacAddressOnPublicNic = true; + // /////////////////////////////////////////////////// // ///////////////// Accessors /////////////////////// // /////////////////////////////////////////////////// @@ -214,6 +219,10 @@ public class CreateVPCCmd extends BaseAsyncCreateCmd implements UserCmd { return BooleanUtils.toBoolean(useVrIpResolver); } + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java index f2327c9073f..6cfdb895977 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmd.java @@ -69,6 +69,11 @@ public class UpdateVPCCmd extends BaseAsyncCustomIdCmd implements UserCmd { since = "4.19") private String sourceNatIP; + @Parameter(name = ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, + type = CommandType.BOOLEAN, since = "4.23.0", authorized = {RoleType.Admin}) + private Boolean keepMacAddressOnPublicNic; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -97,6 +102,10 @@ public class UpdateVPCCmd extends BaseAsyncCustomIdCmd implements UserCmd { return sourceNatIP; } + public Boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java index fdf1e87df50..911839da405 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ExtensionResponse.java @@ -85,6 +85,12 @@ public class ExtensionResponse extends BaseResponse { @Param(description = "Removal timestamp of the extension, if applicable") private Date removed; + @SerializedName(ApiConstants.RESERVED_RESOURCE_DETAILS) + @Param(description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + public ExtensionResponse(String id, String name, String description, String type) { this.id = id; this.name = name; @@ -179,4 +185,8 @@ public class ExtensionResponse extends BaseResponse { public void setRemoved(Date removed) { this.removed = removed; } + + public void setReservedResourceDetails(String reservedResourceDetails) { + this.reservedResourceDetails = reservedResourceDetails; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java index 7c4b733a80f..3a3663af255 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkResponse.java @@ -331,6 +331,10 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement @Param(description = "The BGP peers for the network", since = "4.20.0") private Set bgpPeers; + @SerializedName(ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, since = "4.23.0") + private Boolean keepMacAddressOnPublicNic; + public NetworkResponse() {} public Boolean getDisplayNetwork() { @@ -702,4 +706,8 @@ public class NetworkResponse extends BaseResponseWithAssociatedNetwork implement public void setIpv6Dns2(String ipv6Dns2) { this.ipv6Dns2 = ipv6Dns2; } + + public void setKeepMacAddressOnPublicNic(Boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java index acfabb11350..34d50d5b9f9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpcResponse.java @@ -185,6 +185,10 @@ public class VpcResponse extends BaseResponseWithAnnotations implements Controll @Param(description = "The BGP peers for the VPC", since = "4.20.0") private Set bgpPeers; + @SerializedName(ApiConstants.KEEP_MAC_ADDRESS_ON_PUBLIC_NIC) + @Param(description = ApiConstants.PARAMETER_DESCRIPTION_KEEP_MAC_ADDRESS_ON_PUBLIC_NIC, since = "4.23.0") + private Boolean keepMacAddressOnPublicNic; + public void setId(final String id) { this.id = id; } @@ -366,4 +370,8 @@ public class VpcResponse extends BaseResponseWithAnnotations implements Controll } this.bgpPeers.add(bgpPeer); } + + public void setKeepMacAddressOnPublicNic(Boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java index 6c0121a3e4d..0da11bbd651 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupManager.java @@ -235,7 +235,7 @@ public interface BackupManager extends BackupService, Configurable, PluggableSer * @param forced Indicates if backup will be force removed or not * @return returns operation success */ - boolean deleteBackup(final Long backupId, final Boolean forced); + boolean deleteBackup(final Long backupId, final Boolean forced) throws ResourceAllocationException; void validateBackupForZone(Long zoneId); diff --git a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java index b0fb1ac73c2..d2ebdc25f1b 100644 --- a/api/src/main/java/org/apache/cloudstack/ca/CAManager.java +++ b/api/src/main/java/org/apache/cloudstack/ca/CAManager.java @@ -23,6 +23,8 @@ import java.security.cert.X509Certificate; import java.util.List; import java.util.Map; +import com.trilead.ssh2.Connection; + import org.apache.cloudstack.framework.ca.CAProvider; import org.apache.cloudstack.framework.ca.CAService; import org.apache.cloudstack.framework.ca.Certificate; @@ -39,7 +41,10 @@ public interface CAManager extends CAService, Configurable, PluggableService { ConfigKey CAProviderPlugin = new ConfigKey<>("Advanced", String.class, "ca.framework.provider.plugin", "root", - "The CA provider plugin that is used for secure CloudStack management server-agent communication for encryption and authentication. Restart management server(s) when changed.", true); + "The CA provider plugin used for CloudStack internal certificate management (MS-agent encryption and authentication). " + + "The default 'root' provider auto-generates a CA on first startup, but also supports user-provided custom CA material " + + "via the ca.plugin.root.private.key, ca.plugin.root.public.key, and ca.plugin.root.ca.certificate settings. " + + "Restart management server(s) when changed.", false); ConfigKey CertKeySize = new ConfigKey<>("Advanced", Integer.class, "ca.framework.cert.keysize", @@ -85,6 +90,12 @@ public interface CAManager extends CAService, Configurable, PluggableService { "The actual implementation will depend on the configured CA provider.", false); + ConfigKey CaInjectDefaultTruststore = new ConfigKey<>("Advanced", Boolean.class, + "ca.framework.inject.default.truststore", "true", + "When true, injects the CA provider's certificate into the JVM default truststore on management server startup. " + + "This allows outgoing HTTPS connections from the management server to trust servers with certificates signed by the configured CA. " + + "Restart management server(s) when changed.", false); + /** * Returns a list of available CA provider plugins * @return returns list of CAProvider @@ -130,12 +141,26 @@ public interface CAManager extends CAService, Configurable, PluggableService { boolean revokeCertificate(final BigInteger certSerial, final String certCn, final String provider); /** - * Provisions certificate for given active and connected agent host + * Provisions certificate for given agent host. + * When forced=true, uses SSH to re-provision bypassing the NIO agent connection (for disconnected agents). * @param host + * @param reconnect * @param provider + * @param forced when true, provisions via SSH instead of NIO; supports KVM hosts and SystemVMs * @return returns success/failure as boolean */ - boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider); + boolean provisionCertificate(final Host host, final Boolean reconnect, final String provider, final boolean forced); + + /** + * Provisions certificate for a KVM host using an existing SSH connection. + * Runs keystore-setup to generate a CSR, issues a certificate, then runs keystore-cert-import. + * Used during host discovery and for forced re-provisioning when the NIO agent is unreachable. + * @param sshConnection active SSH connection to the KVM host + * @param agentIp IP address of the KVM host agent + * @param agentHostname hostname of the KVM host agent + * @param caProvider optional CA provider plugin name (null uses default) + */ + void provisionCertificateViaSsh(Connection sshConnection, String agentIp, String agentHostname, String caProvider); /** * Setups up a new keystore and generates CSR for a host diff --git a/api/src/main/java/org/apache/cloudstack/context/CallContext.java b/api/src/main/java/org/apache/cloudstack/context/CallContext.java index 5e0c60184f4..fcfb5b6b1e0 100644 --- a/api/src/main/java/org/apache/cloudstack/context/CallContext.java +++ b/api/src/main/java/org/apache/cloudstack/context/CallContext.java @@ -63,7 +63,7 @@ public class CallContext { private User user; private long userId; private final Map context = new HashMap(); - private final Map apiResourcesUuids = new HashMap<>(); + private final Map apiResourcesUuids = new HashMap<>(); private Project project; private String apiName; @@ -389,11 +389,11 @@ public class CallContext { isEventDisplayEnabled = eventDisplayEnabled; } - public UUID getApiResourceUuid(String paramName) { + public String getApiResourceUuid(String paramName) { return apiResourcesUuids.get(paramName); } - public void putApiResourceUuid(String paramName, UUID uuid) { + public void putApiResourceUuid(String paramName, String uuid) { apiResourcesUuids.put(paramName, uuid); } diff --git a/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java b/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java index f50f841ed74..a01131278a7 100644 --- a/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java +++ b/api/src/main/java/org/apache/cloudstack/extension/ExtensionHelper.java @@ -17,8 +17,11 @@ package org.apache.cloudstack.extension; +import java.util.List; + public interface ExtensionHelper { Long getExtensionIdForCluster(long clusterId); Extension getExtension(long id); Extension getExtensionForCluster(long clusterId); + List getExtensionReservedResourceDetails(long extensionId); } diff --git a/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java b/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java new file mode 100644 index 00000000000..6b3f57b6aa5 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/resourcelimit/Reserver.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.resourcelimit; + +/** + * Interface implemented by CheckedReservation. + *

+ * This is defined in cloud-api to allow methods declared in modules that do not depend on cloud-server + * to receive CheckedReservations as parameters. + */ +public interface Reserver extends AutoCloseable { + + void close(); + +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java index e27ef308d7f..8c164133db8 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/object/BucketApiService.java @@ -95,7 +95,7 @@ public interface BucketApiService { */ Bucket createBucket(CreateBucketCmd cmd); - boolean deleteBucket(long bucketId, Account caller); + boolean deleteBucket(long bucketId, Account caller) throws ResourceAllocationException; boolean updateBucket(UpdateBucketCmd cmd, Account caller) throws ResourceAllocationException; diff --git a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java index 78c598272e0..13f01b4944a 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java @@ -37,7 +37,7 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware); List SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD); + Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint); ConfigKey AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class, "allow.import.volume.with.backing.file", diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java index 4039ca6dc94..38d0df2e8b8 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForGuestNetworkCmdTest.java @@ -40,7 +40,7 @@ public class CreateIpv4SubnetForGuestNetworkCmdTest { @Test public void testCreateIpv4SubnetForGuestNetworkCmd() { Long parentId = 1L; - UUID parentUuid = UUID.randomUUID(); + String parentUuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; Integer cidrSize = 26; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java index bb324aca0e7..560ecdc3b29 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/CreateIpv4SubnetForZoneCmdTest.java @@ -40,7 +40,7 @@ public class CreateIpv4SubnetForZoneCmdTest { @Test public void testCreateIpv4SubnetForZoneCmd() { Long zoneId = 1L; - UUID zoneUuid = UUID.randomUUID(); + String zoneUuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; String accountName = "user"; Long projectId = 10L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java index 31458d2833f..4640510ccda 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DedicateIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class DedicateIpv4SubnetForZoneCmdTest { @Test public void testDedicateIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java index 48aceeaaeec..cd25d8d2401 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForGuestNetworkCmdTest.java @@ -39,7 +39,7 @@ public class DeleteIpv4SubnetForGuestNetworkCmdTest { @Test public void testDeleteIpv4SubnetForGuestNetworkCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteIpv4SubnetForGuestNetworkCmd cmd = new DeleteIpv4SubnetForGuestNetworkCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java index 5c3593a8f1b..269fb3f3c19 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/DeleteIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class DeleteIpv4SubnetForZoneCmdTest { @Test public void testDeleteIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteIpv4SubnetForZoneCmd cmd = new DeleteIpv4SubnetForZoneCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java index 29d6d8e735b..6e5d2a95f3a 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/ReleaseDedicatedIpv4SubnetForZoneCmdTest.java @@ -39,7 +39,7 @@ public class ReleaseDedicatedIpv4SubnetForZoneCmdTest { @Test public void testReleaseDedicatedIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReleaseDedicatedIpv4SubnetForZoneCmd cmd = new ReleaseDedicatedIpv4SubnetForZoneCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java index 399b77de6e8..af37006eafd 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/UpdateIpv4SubnetForZoneCmdTest.java @@ -40,7 +40,7 @@ public class UpdateIpv4SubnetForZoneCmdTest { @Test public void testUpdateIpv4SubnetForZoneCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String subnet = "192.168.1.0/24"; UpdateIpv4SubnetForZoneCmd cmd = new UpdateIpv4SubnetForZoneCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java index 9cd403ddd1d..3db1fab466f 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForNetworkCmdTest.java @@ -46,7 +46,7 @@ public class ChangeBgpPeersForNetworkCmdTest { @Test public void testChangeBgpPeersForNetworkCmd() { Long networkId = 10L; - UUID networkUuid = UUID.randomUUID(); + String networkUuid = UUID.randomUUID().toString(); List bgpPeerIds = Arrays.asList(20L, 21L); ChangeBgpPeersForNetworkCmd cmd = new ChangeBgpPeersForNetworkCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java index 545523e3ab9..fb85f706068 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ChangeBgpPeersForVpcCmdTest.java @@ -46,7 +46,7 @@ public class ChangeBgpPeersForVpcCmdTest { @Test public void testChangeBgpPeersForVpcCmd() { Long VpcId = 10L; - UUID vpcUuid = UUID.randomUUID(); + String vpcUuid = UUID.randomUUID().toString(); List bgpPeerIds = Arrays.asList(20L, 21L); ChangeBgpPeersForVpcCmd cmd = new ChangeBgpPeersForVpcCmd(); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java index 866824f6293..3abc5f57d01 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/CreateBgpPeerCmdTest.java @@ -40,7 +40,7 @@ public class CreateBgpPeerCmdTest { @Test public void testCreateBgpPeerCmd() { Long zoneId = 1L; - UUID zoneUuid = UUID.randomUUID(); + String zoneUuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java index a8046d3d745..c5edb1b8f53 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DedicateBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class DedicateBgpPeerCmdTest { @Test public void testDedicateBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String accountName = "user"; Long projectId = 10L; Long domainId = 11L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java index d54be9e859e..5228a63dc92 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/DeleteBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class DeleteBgpPeerCmdTest { @Test public void testDeleteBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); DeleteBgpPeerCmd cmd = new DeleteBgpPeerCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java index 1cf8ead706d..60a814d6305 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/ReleaseDedicatedBgpPeerCmdTest.java @@ -39,7 +39,7 @@ public class ReleaseDedicatedBgpPeerCmdTest { @Test public void testReleaseDedicatedBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReleaseDedicatedBgpPeerCmd cmd = new ReleaseDedicatedBgpPeerCmd(); ReflectionTestUtils.setField(cmd, "id", id); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java index 1601fcb4c5a..d594bc5718b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/network/bgp/UpdateBgpPeerCmdTest.java @@ -40,7 +40,7 @@ public class UpdateBgpPeerCmdTest { @Test public void testUpdateBgpPeerCmd() { Long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); String ip4Address = "ip4-address"; String ip6Address = "ip6-address"; Long peerAsNumber = 15000L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java index 3b2b49cae4e..8d2771c969b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java @@ -97,7 +97,7 @@ public class DownloadImageStoreObjectCmdTest { @Test public void testGetEventDescription() { - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); ReflectionTestUtils.setField(cmd, "storeId", 1L); ReflectionTestUtils.setField(cmd, "path", "path/to/object"); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java index ecca507a6b9..59a61806e86 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/volume/UnmanageVolumeCmdTest.java @@ -44,7 +44,7 @@ public class UnmanageVolumeCmdTest { public void testUnmanageVolumeCmd() { long accountId = 2L; Long volumeId = 3L; - UUID volumeUuid = UUID.randomUUID(); + String volumeUuid = UUID.randomUUID().toString(); Volume volume = Mockito.mock(Volume.class); Mockito.when(responseGenerator.findVolumeById(volumeId)).thenReturn(volume); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java index f100822b8c7..cd0390aa268 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/AddAccountToProjectCmdTest.java @@ -16,6 +16,7 @@ // under the License. package org.apache.cloudstack.api.command.test; +import com.cloud.exception.ResourceAllocationException; import junit.framework.Assert; import junit.framework.TestCase; @@ -149,6 +150,8 @@ public class AddAccountToProjectCmdTest extends TestCase { addAccountToProjectCmd.execute(); } catch (InvalidParameterValueException exception) { Assert.assertEquals("Either accountName or email is required", exception.getLocalizedMessage()); + } catch (ResourceAllocationException exception) { + Assert.fail(); } } diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java index b70efaf9a6c..032dca8d800 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/CreateSnapshotCmdTest.java @@ -118,7 +118,7 @@ public class CreateSnapshotCmdTest extends TestCase { AccountService accountService = Mockito.mock(AccountService.class); Account account = Mockito.mock(Account.class); Mockito.when(accountService.getAccount(anyLong())).thenReturn(account); - UUID volumeUuid = UUID.randomUUID(); + String volumeUuid = UUID.randomUUID().toString(); CallContext.current().putApiResourceUuid("volumeid", volumeUuid); diff --git a/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java index 3dfb29dadd3..c78dbe9b56b 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/test/UpdateConditionCmdTest.java @@ -56,7 +56,7 @@ public class UpdateConditionCmdTest { private static final Long threshold = 100L; private static final long accountId = 5L; - private static final UUID conditionUuid = UUID.randomUUID(); + private static final String conditionUuid = UUID.randomUUID().toString(); @Before public void setUp() { diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java index d3cf5dd6cd6..dbe7669431d 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/network/routing/DeleteRoutingFirewallRuleCmdTest.java @@ -49,7 +49,7 @@ public class DeleteRoutingFirewallRuleCmdTest { ReflectionTestUtils.setField(cmd, "_firewallService", _firewallService); long id = 1L; - UUID uuid = UUID.randomUUID(); + String uuid = UUID.randomUUID().toString(); long accountId = 2L; long networkId = 3L; diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java index acb2dc68597..9d56cffd671 100644 --- a/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/vpc/UpdateVPCCmdTest.java @@ -93,7 +93,7 @@ public class UpdateVPCCmdTest extends TestCase { responseGenerator = Mockito.mock(ResponseGenerator.class); cmd._responseGenerator = responseGenerator; Mockito.verify(_vpcService, Mockito.times(0)).updateVpc(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(), Mockito.anyString()); + Mockito.anyString(), Mockito.anyBoolean(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyBoolean()); } } diff --git a/client/pom.xml b/client/pom.xml index b8dffe65d4f..ba16cc5d34a 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -121,6 +121,11 @@ cloud-plugin-storage-volume-adaptive ${project.version} + + org.apache.cloudstack + cloud-plugin-storage-volume-ontap + ${project.version} + org.apache.cloudstack cloud-plugin-storage-volume-solidfire @@ -372,11 +377,6 @@ cloud-plugin-explicit-dedication ${project.version} - - org.apache.cloudstack - cloud-plugin-host-allocator-random - ${project.version} - org.apache.cloudstack cloud-plugin-outofbandmanagement-driver-ipmitool @@ -716,17 +716,17 @@ org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on ${cs.bcprov.version} org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on ${cs.bcprov.version} org.bouncycastle - bctls-jdk15on + bctls-jdk18on ${cs.bcprov.version} @@ -906,13 +906,13 @@ org.bouncycastle - bcprov-jdk15on + bcprov-jdk18on false ${project.build.directory}/lib org.bouncycastle - bcpkix-jdk15on + bcpkix-jdk18on false ${project.build.directory}/lib @@ -936,7 +936,7 @@ org.bouncycastle - bctls-jdk15on + bctls-jdk18on false ${project.build.directory}/lib @@ -971,9 +971,9 @@ org.apache.tomcat.embed:tomcat-embed-core org.apache.geronimo.specs:geronimo-servlet_3.0_spec org.apache.geronimo.specs:geronimo-javamail_1.4_spec - org.bouncycastle:bcprov-jdk15on - org.bouncycastle:bcpkix-jdk15on - org.bouncycastle:bctls-jdk15on + org.bouncycastle:bcprov-jdk18on + org.bouncycastle:bcpkix-jdk18on + org.bouncycastle:bctls-jdk18on com.mysql:mysql-connector-j org.apache.cloudstack:cloud-plugin-storage-volume-storpool org.apache.cloudstack:cloud-plugin-storage-volume-linstor diff --git a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java index fc066e5c589..c46fb697a3c 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckConvertInstanceCommand.java @@ -18,6 +18,8 @@ package com.cloud.agent.api; public class CheckConvertInstanceCommand extends Command { boolean checkWindowsGuestConversionSupport = false; + boolean useVddk = false; + String vddkLibDir; public CheckConvertInstanceCommand() { } @@ -26,6 +28,11 @@ public class CheckConvertInstanceCommand extends Command { this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; } + public CheckConvertInstanceCommand(boolean checkWindowsGuestConversionSupport, boolean useVddk) { + this.checkWindowsGuestConversionSupport = checkWindowsGuestConversionSupport; + this.useVddk = useVddk; + } + @Override public boolean executeInSequence() { return false; @@ -34,4 +41,20 @@ public class CheckConvertInstanceCommand extends Command { public boolean getCheckWindowsGuestConversionSupport() { return checkWindowsGuestConversionSupport; } + + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } } diff --git a/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java b/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java index 5a26b22ec6a..41e266784db 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/CheckOnHostAnswer.java @@ -38,6 +38,8 @@ public class CheckOnHostAnswer extends Answer { public CheckOnHostAnswer(CheckOnHostCommand cmd, String details) { super(cmd, false, details); + determined = false; + alive = false; } public boolean isDetermined() { @@ -47,5 +49,4 @@ public class CheckOnHostAnswer extends Answer { public boolean isAlive() { return alive; } - } diff --git a/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java b/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java index 94239f2900e..72b5217604d 100644 --- a/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java +++ b/core/src/main/java/com/cloud/agent/api/CheckOnHostCommand.java @@ -24,7 +24,7 @@ import com.cloud.host.Host; public class CheckOnHostCommand extends Command { HostTO host; - boolean reportCheckFailureIfOneStorageIsDown; + boolean reportIfHeartBeatFailedForOneStoragePool; protected CheckOnHostCommand() { } @@ -34,17 +34,17 @@ public class CheckOnHostCommand extends Command { setWait(20); } - public CheckOnHostCommand(Host host, boolean reportCheckFailureIfOneStorageIsDown) { + public CheckOnHostCommand(Host host, boolean reportIfHeartBeatFailedForOneStoragePool) { this(host); - this.reportCheckFailureIfOneStorageIsDown = reportCheckFailureIfOneStorageIsDown; + this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool; } public HostTO getHost() { return host; } - public boolean isCheckFailedOnOneStorage() { - return reportCheckFailureIfOneStorageIsDown; + public boolean shouldReportIfHeartBeatFailedForOneStoragePool() { + return reportIfHeartBeatFailedForOneStoragePool; } @Override diff --git a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java index 24336747ccf..38e0dca7736 100644 --- a/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ConvertInstanceCommand.java @@ -31,6 +31,10 @@ public class ConvertInstanceCommand extends Command { private boolean exportOvfToConversionLocation; private int threadsCountToExportOvf = 0; private String extraParams; + private boolean useVddk; + private String vddkLibDir; + private String vddkTransports; + private String vddkThumbprint; public ConvertInstanceCommand() { } @@ -90,6 +94,38 @@ public class ConvertInstanceCommand extends Command { this.extraParams = extraParams; } + public boolean isUseVddk() { + return useVddk; + } + + public void setUseVddk(boolean useVddk) { + this.useVddk = useVddk; + } + + public String getVddkLibDir() { + return vddkLibDir; + } + + public void setVddkLibDir(String vddkLibDir) { + this.vddkLibDir = vddkLibDir; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public void setVddkTransports(String vddkTransports) { + this.vddkTransports = vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public void setVddkThumbprint(String vddkThumbprint) { + this.vddkThumbprint = vddkThumbprint; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java index ed337885bee..21c4e7b97d0 100644 --- a/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java +++ b/core/src/main/java/com/cloud/agent/api/PropagateResourceEventCommand.java @@ -24,6 +24,8 @@ import com.cloud.resource.ResourceState; public class PropagateResourceEventCommand extends Command { long hostId; ResourceState.Event event; + boolean forced; + boolean forceDeleteStorage; protected PropagateResourceEventCommand() { @@ -34,6 +36,13 @@ public class PropagateResourceEventCommand extends Command { this.event = event; } + public PropagateResourceEventCommand(long hostId, ResourceState.Event event, boolean forced, boolean forceDeleteStorage) { + this.hostId = hostId; + this.event = event; + this.forced = forced; + this.forceDeleteStorage = forceDeleteStorage; + } + public long getHostId() { return hostId; } @@ -42,6 +51,14 @@ public class PropagateResourceEventCommand extends Command { return event; } + public boolean isForced() { + return forced; + } + + public boolean isForceDeleteStorage() { + return forceDeleteStorage; + } + @Override public boolean executeInSequence() { // TODO Auto-generated method stub diff --git a/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java b/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java index dc63b1ee746..e2953e9c7ad 100644 --- a/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java +++ b/core/src/main/java/com/cloud/agent/api/ScaleVmCommand.java @@ -30,6 +30,7 @@ public class ScaleVmCommand extends Command { Integer maxSpeed; long minRam; long maxRam; + private boolean limitCpuUseChange; public VirtualMachineTO getVm() { return vm; @@ -43,7 +44,7 @@ public class ScaleVmCommand extends Command { return cpus; } - public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse) { + public ScaleVmCommand(String vmName, int cpus, Integer minSpeed, Integer maxSpeed, long minRam, long maxRam, boolean limitCpuUse, Double cpuQuotaPercentage, boolean limitCpuUseChange) { super(); this.vmName = vmName; this.cpus = cpus; @@ -52,6 +53,8 @@ public class ScaleVmCommand extends Command { this.minRam = minRam; this.maxRam = maxRam; this.vm = new VirtualMachineTO(1L, vmName, null, cpus, minSpeed, maxSpeed, minRam, maxRam, null, null, false, limitCpuUse, null); + this.vm.setCpuQuotaPercentage(cpuQuotaPercentage); + this.limitCpuUseChange = limitCpuUseChange; } public void setCpus(int cpus) { @@ -102,6 +105,10 @@ public class ScaleVmCommand extends Command { return vm; } + public boolean getLimitCpuUseChange() { + return limitCpuUseChange; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java index d8cc74817d7..96d73e11990 100644 --- a/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java +++ b/core/src/main/java/com/cloud/agent/api/routing/LoadBalancerConfigCommand.java @@ -36,6 +36,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { public String lbStatsAuth = "admin1:AdMiN123"; public String lbStatsUri = "/admin?stats"; public String maxconn = ""; + public Long idleTimeout = 50000L; /* 0=infinite, >0 = timeout in milliseconds */ public String lbProtocol; public boolean keepAliveEnabled = false; NicTO nic; @@ -50,7 +51,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { } public LoadBalancerConfigCommand(LoadBalancerTO[] loadBalancers, String publicIp, String guestIp, String privateIp, NicTO nic, Long vpcId, String maxconn, - boolean keepAliveEnabled) { + boolean keepAliveEnabled, Long idleTimeout) { this.loadBalancers = loadBalancers; this.lbStatsPublicIP = publicIp; this.lbStatsPrivateIP = privateIp; @@ -59,6 +60,7 @@ public class LoadBalancerConfigCommand extends NetworkElementCommand { this.vpcId = vpcId; this.maxconn = maxconn; this.keepAliveEnabled = keepAliveEnabled; + this.idleTimeout = idleTimeout; } public NicTO getNic() { diff --git a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java index 0c6373134b1..1c5eb7b9a9a 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DownloadAnswer.java @@ -140,7 +140,7 @@ public class DownloadAnswer extends Answer { } public Long getTemplateSize() { - return templateSize; + return templateSize == 0 ? templatePhySicalSize : templateSize; } public void setTemplatePhySicalSize(long templatePhySicalSize) { diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index 128652fc64f..7d544c2e49c 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -635,6 +635,19 @@ public class HAProxyConfigurator implements LoadBalancerConfigurator { if (lbCmd.keepAliveEnabled) { dSection.set(7, "\tno option httpclose"); } + if (lbCmd.idleTimeout > 0) { + dSection.set(9, "\ttimeout client " + Long.toString(lbCmd.idleTimeout)); + dSection.set(10, "\ttimeout server " + Long.toString(lbCmd.idleTimeout)); + } else if (lbCmd.idleTimeout == 0) { + // .remove() is not allowed, only .set() operations are allowed as the list + // is a fixed size. So lets just mark the entry as blank. + dSection.set(9, ""); + dSection.set(10, ""); + } else { + // Negative idleTimeout values are considered invalid; retain the + // default HAProxy timeout values from defaultsSection for predictability. + logger.warn("Negative idleTimeout ({}) configured; retaining default HAProxy timeouts.", lbCmd.idleTimeout); + } if (logger.isDebugEnabled()) { for (final String s : dSection) { diff --git a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java index 6fe001de72c..71c329796d1 100755 --- a/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java @@ -52,6 +52,7 @@ import com.cloud.storage.StorageLayer; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.net.Proxy; /** @@ -125,6 +126,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } diff --git a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java index 95ed0d1e76d..0dad2564779 100644 --- a/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/MetalinkTemplateDownloader.java @@ -18,8 +18,11 @@ // package com.cloud.storage.template; + import com.cloud.storage.StorageLayer; import com.cloud.utils.UriUtils; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; + import org.apache.commons.httpclient.HttpClient; import org.apache.commons.httpclient.HttpMethod; import org.apache.commons.httpclient.HttpMethodRetryHandler; @@ -59,6 +62,7 @@ public class MetalinkTemplateDownloader extends TemplateDownloaderBase implement GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, myretryhandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (!toFileSet) { String[] parts = downloadUrl.split("/"); String filename = parts[parts.length - 1]; diff --git a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java index 8719947cb4f..6608754073a 100644 --- a/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java +++ b/core/src/main/java/com/cloud/storage/template/SimpleHttpMultiFileDownloader.java @@ -44,6 +44,7 @@ import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.lang3.StringUtils; import com.cloud.storage.StorageLayer; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implements TemplateDownloader { private static final MultiThreadedHttpConnectionManager s_httpClientManager = new MultiThreadedHttpConnectionManager(); @@ -95,6 +96,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem GetMethod request = new GetMethod(downloadUrl); request.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryHandler); request.setFollowRedirects(followRedirects); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); return request; } @@ -141,6 +143,7 @@ public class SimpleHttpMultiFileDownloader extends ManagedContextRunnable implem continue; } HeadMethod headMethod = new HeadMethod(downloadUrl); + headMethod.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { if (client.executeMethod(headMethod) != HttpStatus.SC_OK) { continue; diff --git a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java index f5ad5fbea2c..972c2eaf7bb 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/RestoreBackupCommand.java @@ -34,6 +34,7 @@ public class RestoreBackupCommand extends Command { private List backupVolumesUUIDs; private List restoreVolumePools; private List restoreVolumePaths; + private List restoreVolumeSizes; private List backupFiles; private String diskType; private Boolean vmExists; @@ -92,6 +93,14 @@ public class RestoreBackupCommand extends Command { this.restoreVolumePaths = restoreVolumePaths; } + public List getRestoreVolumeSizes() { + return restoreVolumeSizes; + } + + public void setRestoreVolumeSizes(List restoreVolumeSizes) { + this.restoreVolumeSizes = restoreVolumeSizes; + } + public List getBackupFiles() { return backupFiles; } diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java index a1485463eaa..05619e5632b 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/DirectTemplateDownloaderImpl.java @@ -21,6 +21,7 @@ package org.apache.cloudstack.direct.download; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -33,6 +34,7 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.UUID; public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDownloader { @@ -128,15 +130,14 @@ public abstract class DirectTemplateDownloaderImpl implements DirectTemplateDown */ protected File createTemporaryDirectoryAndFile(String downloadDir) { createFolder(downloadDir); - return new File(downloadDir + File.separator + getFileNameFromUrl()); + return new File(downloadDir + File.separator + getTemporaryFileName()); } /** - * Return filename from url + * Return filename from the temporary download file */ - public String getFileNameFromUrl() { - String[] urlParts = url.split("/"); - return urlParts[urlParts.length - 1]; + public String getTemporaryFileName() { + return String.format("%s.%s", UUID.randomUUID(), FilenameUtils.getExtension(url)); } @Override diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java index c4a802ecdbc..99b84bb645c 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/HttpDirectTemplateDownloader.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.direct.download; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -32,6 +33,7 @@ import java.util.Map; import com.cloud.utils.Pair; import com.cloud.utils.UriUtils; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.net.HttpClientCloudStackUserAgent; import com.cloud.utils.storage.QCOW2Utils; import org.apache.commons.collections.MapUtils; import org.apache.commons.httpclient.HttpClient; @@ -39,6 +41,7 @@ import org.apache.commons.httpclient.HttpStatus; import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.HeadMethod; +import org.apache.commons.httpclient.params.HttpMethodParams; import org.apache.commons.io.IOUtils; public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { @@ -68,6 +71,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { protected GetMethod createRequest(String downloadUrl, Map headers) { GetMethod request = new GetMethod(downloadUrl); request.setFollowRedirects(this.isFollowRedirects()); + request.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); if (MapUtils.isNotEmpty(headers)) { for (String key : headers.keySet()) { request.setRequestHeader(key, headers.get(key)); @@ -111,6 +115,7 @@ public class HttpDirectTemplateDownloader extends DirectTemplateDownloaderImpl { public boolean checkUrl(String url) { HeadMethod httpHead = new HeadMethod(url); httpHead.setFollowRedirects(this.isFollowRedirects()); + httpHead.getParams().setParameter(HttpMethodParams.USER_AGENT, HttpClientCloudStackUserAgent.CLOUDSTACK_USER_AGENT); try { int responseCode = client.executeMethod(httpHead); if (responseCode != HttpStatus.SC_OK) { diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java index 2050b9ef09f..854c310cde9 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/MetalinkDirectTemplateDownloader.java @@ -97,7 +97,7 @@ public class MetalinkDirectTemplateDownloader extends DirectTemplateDownloaderIm DirectTemplateDownloader urlDownloader = createDownloaderForMetalinks(getUrl(), getTemplateId(), getDestPoolPath(), getChecksum(), headers, connectTimeout, soTimeout, null, temporaryDownloadPath); try { - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getTemporaryFileName()); File f = new File(getDownloadedFilePath()); if (f.exists()) { f.delete(); diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java index 21184ef07fe..6b0959b78ff 100644 --- a/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java +++ b/core/src/main/java/org/apache/cloudstack/direct/download/NfsDirectTemplateDownloader.java @@ -69,7 +69,7 @@ public class NfsDirectTemplateDownloader extends DirectTemplateDownloaderImpl { String mount = String.format(mountCommand, srcHost + ":" + srcPath, "/mnt/" + mountSrcUuid); Script.runSimpleBashScript(mount); String downloadDir = getDestPoolPath() + File.separator + getDirectDownloadTempPath(getTemplateId()); - setDownloadedFilePath(downloadDir + File.separator + getFileNameFromUrl()); + setDownloadedFilePath(downloadDir + File.separator + getTemporaryFileName()); Script.runSimpleBashScript("cp /mnt/" + mountSrcUuid + srcPath + " " + getDownloadedFilePath()); Script.runSimpleBashScript("umount /mnt/" + mountSrcUuid); return new Pair<>(true, getDownloadedFilePath()); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index 253a2607a72..98db75f3902 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -19,6 +19,9 @@ package org.apache.cloudstack.storage.command; +import com.cloud.configuration.Resource; +import org.apache.cloudstack.utils.bytescale.ByteScaleUtils; + public class TemplateOrVolumePostUploadCommand { long entityId; @@ -185,6 +188,11 @@ public class TemplateOrVolumePostUploadCommand { this.description = description; } + public void setDefaultMaxSecondaryStorageInBytes(long defaultMaxSecondaryStorageInBytes) { + this.defaultMaxSecondaryStorageInGB = defaultMaxSecondaryStorageInBytes != Resource.RESOURCE_UNLIMITED ? + ByteScaleUtils.bytesToGibibytes(defaultMaxSecondaryStorageInBytes) : Resource.RESOURCE_UNLIMITED; + } + public void setDefaultMaxSecondaryStorageInGB(long defaultMaxSecondaryStorageInGB) { this.defaultMaxSecondaryStorageInGB = defaultMaxSecondaryStorageInGB; } diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java index 9e6b76e467f..f78744046f7 100644 --- a/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java +++ b/core/src/main/java/org/apache/cloudstack/storage/command/UploadStatusCommand.java @@ -28,6 +28,7 @@ public class UploadStatusCommand extends Command { } private String entityUuid; private EntityType entityType; + private Boolean abort; protected UploadStatusCommand() { } @@ -37,6 +38,11 @@ public class UploadStatusCommand extends Command { this.entityType = entityType; } + public UploadStatusCommand(String entityUuid, EntityType entityType, Boolean abort) { + this(entityUuid, entityType); + this.abort = abort; + } + public String getEntityUuid() { return entityUuid; } @@ -45,6 +51,10 @@ public class UploadStatusCommand extends Command { return entityType; } + public Boolean getAbort() { + return abort; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java index 6d4c6234c42..4ddadab999a 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/ConfigHelperTest.java @@ -235,7 +235,7 @@ public class ConfigHelperTest { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 0L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); diff --git a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java index 4196587cc3f..ed819bb7c68 100644 --- a/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java +++ b/core/src/test/java/com/cloud/agent/resource/virtualnetwork/VirtualRoutingResourceTest.java @@ -779,7 +779,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { final LoadBalancerTO[] arrayLbs = new LoadBalancerTO[lbs.size()]; lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, null, "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; @@ -795,7 +795,7 @@ public class VirtualRoutingResourceTest implements VirtualRouterDeployer { lbs.toArray(arrayLbs); final NicTO nic = new NicTO(); nic.setIp("10.1.10.2"); - final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false); + final LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(arrayLbs, "64.10.2.10", "10.1.10.2", "192.168.1.2", nic, Long.valueOf(1), "1000", false, 50000L); cmd.setAccessDetail(NetworkElementCommand.ROUTER_IP, "10.1.10.2"); cmd.setAccessDetail(NetworkElementCommand.ROUTER_NAME, ROUTERNAME); return cmd; diff --git a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java index 72361c2880e..073f976719b 100644 --- a/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java +++ b/core/src/test/java/com/cloud/network/HAProxyConfiguratorTest.java @@ -79,13 +79,14 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("keepalive disabled should result in 'option httpclose' in the resulting haproxy config", result.contains("\toption httpclose")); - cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true); + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); result = genConfig(hpg, cmd); assertTrue("keepalive enabled should result in 'no option httpclose' in the resulting haproxy config", result.contains("\tno option httpclose")); + // TODO // create lb command // setup tests for @@ -93,6 +94,27 @@ public class HAProxyConfiguratorTest { // httpmode } + /** + * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. + */ + @Test + public void testGenerateConfigurationLoadBalancerIdleTimeoutConfigCommand() { + LoadBalancerTO lb = new LoadBalancerTO("1", "10.2.0.1", 80, "http", "bla", false, false, false, null); + LoadBalancerTO[] lba = new LoadBalancerTO[1]; + lba[0] = lb; + HAProxyConfigurator hpg = new HAProxyConfigurator(); + + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 0L); + String result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 0 should not generate 'timeout server' in the resulting haproxy config", !result.contains("\ttimeout server")); + assertTrue("idleTimeout of 0 should not generate 'timeout client' in the resulting haproxy config", !result.contains("\ttimeout client")); + + cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "4", true, 1234L); + result = genConfig(hpg, cmd); + assertTrue("idleTimeout of 1234 should result in 'timeout server 1234' in the resulting haproxy config", result.contains("\ttimeout server 1234")); + assertTrue("idleTimeout of 1234 should result in 'timeout client 1234' in the resulting haproxy config", result.contains("\ttimeout client 1234")); + } + /** * Test method for {@link com.cloud.network.HAProxyConfigurator#generateConfiguration(com.cloud.agent.api.routing.LoadBalancerConfigCommand)}. */ @@ -106,7 +128,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); assertTrue("'send-proxy' should result if protocol is 'tcp-proxy'", result.contains("send-proxy")); } @@ -118,7 +140,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("acl network_allowed src 1.1.1.1 2.2.2.2/24 \n\ttcp-request connection reject if !network_allowed")); } @@ -131,7 +153,7 @@ public class HAProxyConfiguratorTest { LoadBalancerTO[] lba = new LoadBalancerTO[1]; lba[0] = lb; HAProxyConfigurator hpg = new HAProxyConfigurator(); - LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false); + LoadBalancerConfigCommand cmd = new LoadBalancerConfigCommand(lba, "10.0.0.1", "10.1.0.1", "10.1.1.1", null, 1L, "12", false, 0L); String result = genConfig(hpg, cmd); Assert.assertTrue(result.contains("bind 10.2.0.1:443 ssl crt /etc/cloudstack/ssl/10_2_0_1-443.pem")); } diff --git a/debian/changelog b/debian/changelog index 02251137e9d..41f97748a0b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -2,14 +2,20 @@ cloudstack (4.23.0.0-SNAPSHOT) unstable; urgency=low * Update the version to 4.23.0.0-SNAPSHOT + -- the Apache CloudStack project Fri, 22 May 2026 10:20:00 -0300 + +cloudstack (4.22.1.0) unstable; urgency=low + + * Update the version to 4.22.1.0 + + -- the Apache CloudStack project Mon, 11 May 2026 20:26:07 +0530 + +cloudstack (4.22.0.0) unstable; urgency=low + + * Update the version to 4.22.0.0 + -- the Apache CloudStack project Thu, 30 Oct 2025 19:23:55 +0530 -cloudstack (4.23.0.0-SNAPSHOT-SNAPSHOT) unstable; urgency=low - - * Update the version to 4.23.0.0-SNAPSHOT-SNAPSHOT - - -- the Apache CloudStack project Thu, Aug 28 11:58:36 2025 +0530 - cloudstack (4.21.0.0) unstable; urgency=low * Update the version to 4.21.0.0 diff --git a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java index 57dc1b7bf72..e871bd8672f 100644 --- a/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java +++ b/engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java @@ -181,15 +181,6 @@ public interface VirtualMachineManager extends Manager { void advanceReboot(String vmUuid, Map params) throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException, OperationTimedoutException; - /** - * Check to see if a virtual machine can be upgraded to the given service offering - * - * @param vm - * @param offering - * @return true if the host can handle the upgrade, false otherwise - */ - boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering); - VirtualMachine findById(long vmId); void storageMigration(String vmUuid, Map volumeToPool); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java index 947cbd8e618..030c1277efe 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java @@ -122,6 +122,14 @@ public interface NetworkOrchestrationService { "Load Balancer(haproxy) maximum number of concurrent connections(global max)", true, Scope.Global); + ConfigKey NETWORK_LB_HAPROXY_IDLE_TIMEOUT = new ConfigKey<>( + "Network", + Long.class, + "network.loadbalancer.haproxy.idle.timeout", + "50000", + "Load Balancer(haproxy) idle timeout in milliseconds. Use 0 for infinite.", + true, + Scope.Global); List setupNetwork(Account owner, NetworkOffering offering, DeploymentPlan plan, String name, String displayText, boolean isDefault) throws ConcurrentOperationException; @@ -212,6 +220,11 @@ public interface NetworkOrchestrationService { Boolean displayNetworkEnabled, String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + Network createGuestNetwork(long networkOfferingId, String name, String displayText, String gateway, String cidr, String vlanId, boolean bypassVlanOverlapCheck, String networkDomain, Account owner, + Long domainId, PhysicalNetwork physicalNetwork, long zoneId, ACLType aclType, Boolean subdomainAccess, Long vpcId, String ip6Gateway, String ip6Cidr, + Boolean displayNetworkEnabled, String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, + String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize, boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException; + UserDataServiceProvider getPasswordResetProvider(Network network); UserDataServiceProvider getSSHKeyResetProvider(Network network); @@ -310,7 +323,7 @@ public interface NetworkOrchestrationService { void removeDhcpServiceInSubnet(Nic nic); - boolean resourceCountNeedsUpdate(NetworkOffering ntwkOff, ACLType aclType); + boolean isResourceCountUpdateNeeded(NetworkOffering networkOffering); void prepareAllNicsForMigration(VirtualMachineProfile vm, DeployDestination dest); diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 6f8c4630456..a55219511cb 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -120,7 +120,7 @@ public interface VolumeOrchestrationService { void destroyVolume(Volume volume); DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId); + Account owner, Long deviceId, boolean incrementResourceCount); VolumeInfo createVolumeOnPrimaryStorage(VirtualMachine vm, VolumeInfo volume, HypervisorType rootDiskHyperType, StoragePool storagePool) throws NoTransitionException; diff --git a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java index 4c81c7359f2..abaf6ea967d 100644 --- a/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java +++ b/engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java @@ -133,6 +133,20 @@ public interface CapacityManager { "capacity.calculate.workers", "1", "Number of worker threads to be used for capacities calculation", true); + ConfigKey KvmMemoryDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "kvm.memory.dynamic.scaling.capacity", "0", + "Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. " + + "The 'kvm.memory.dynamic.scaling.capacity' setting's value will be used to define the value of the " + + "'' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's memory capacity will be considered.", + true, ConfigKey.Scope.Cluster); + + ConfigKey KvmCpuDynamicScalingCapacity = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "kvm.cpu.dynamic.scaling.capacity", "0", + "Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. " + + "The 'kvm.cpu.dynamic.scaling.capacity' setting's value will be used to define the value of the " + + "'' element of domain XMLs. If it is set to a value less than or equal to '0', then the host's CPU cores capacity will be considered.", + true, ConfigKey.Scope.Cluster); + public boolean releaseVmCapacity(VirtualMachine vm, boolean moveFromReserved, boolean moveToReservered, Long hostId); void allocateVmCapacity(VirtualMachine vm, boolean fromLastHost); diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index ddc8153d739..53bfcce2703 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -21,6 +21,7 @@ import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Cluster; import com.cloud.deploy.DeploymentPlanner; import com.cloud.host.HostVO; import com.cloud.host.Status; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.component.Manager; import com.cloud.vm.VMInstanceVO; import org.apache.cloudstack.framework.config.ConfigKey; @@ -32,6 +33,8 @@ import java.util.List; */ public interface HighAvailabilityManager extends Manager { + List LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT = List.of(StoragePoolType.NetworkFilesystem, StoragePoolType.SharedMountPoint); + ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); @@ -72,10 +75,10 @@ public interface HighAvailabilityManager extends Manager { + " which are registered for the HA event that were successful and are now ready to be purged.", true, Cluster); - public static final ConfigKey KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false", + ConfigKey KvmHAFenceHostIfHeartbeatFailsOnStorage = new ConfigKey<>("Advanced", Boolean.class, "kvm.ha.fence.on.storage.heartbeat.failure", "false", "Proceed fencing the host even the heartbeat failed for only one storage pool", false, ConfigKey.Scope.Zone); - public enum WorkType { + enum WorkType { Migration, // Migrating VMs off of a host. Stop, // Stops a VM for storage pool migration purposes. This should be obsolete now. CheckStop, // Checks if a VM has been stopped. diff --git a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java index e724f5d081b..4767e86e8ab 100755 --- a/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java +++ b/engine/components-api/src/main/java/com/cloud/resource/ResourceManager.java @@ -122,6 +122,8 @@ public interface ResourceManager extends ResourceService, Configurable { public boolean executeUserRequest(long hostId, ResourceState.Event event) throws AgentUnavailableException; + boolean executeUserRequest(long hostId, ResourceState.Event event, boolean isForced, boolean isForceDeleteStorage) throws AgentUnavailableException; + boolean resourceStateTransitTo(Host host, Event event, long msId) throws NoTransitionException; boolean umanageHost(long hostId); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 527c5259376..1215829d92f 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -805,8 +805,11 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); + String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT); + String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR); + String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion)) { + if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { _hostDao.loadDetails(host); boolean updateNeeded = false; if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { @@ -821,6 +824,26 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl host.getDetails().put(Host.HOST_OVFTOOL_VERSION, ovftoolVersion); updateNeeded = true; } + if (StringUtils.isNotBlank(vddkSupport) && !vddkSupport.equals(host.getDetails().get(Host.HOST_VDDK_SUPPORT))) { + host.getDetails().put(Host.HOST_VDDK_SUPPORT, vddkSupport); + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkLibDir).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_LIB_DIR)))) { + if (StringUtils.isBlank(vddkLibDir)) { + host.getDetails().remove(Host.HOST_VDDK_LIB_DIR); + } else { + host.getDetails().put(Host.HOST_VDDK_LIB_DIR, vddkLibDir); + } + updateNeeded = true; + } + if (!StringUtils.defaultString(vddkVersion).equals(StringUtils.defaultString(host.getDetails().get(Host.HOST_VDDK_VERSION)))) { + if (StringUtils.isBlank(vddkVersion)) { + host.getDetails().remove(Host.HOST_VDDK_VERSION); + } else { + host.getDetails().put(Host.HOST_VDDK_VERSION, vddkVersion); + } + updateNeeded = true; + } if (updateNeeded) { _hostDao.saveDetails(host); } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java index cfa0949883f..38a198b7304 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java @@ -1306,11 +1306,20 @@ public class ClusteredAgentManagerImpl extends AgentManagerImpl implements Clust boolean result; try { - result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent()); + result = _resourceMgr.executeUserRequest(cmd.getHostId(), cmd.getEvent(), cmd.isForced(), cmd.isForceDeleteStorage()); logger.debug("Result is {}", result); } catch (final AgentUnavailableException ex) { logger.warn("Agent is unavailable", ex); return null; + } catch (final RuntimeException ex) { + logger.error(String.format("Failed to execute propagated event %s for host %d", cmd.getEvent().name(), cmd.getHostId()), ex); + final Answer[] answers = new Answer[1]; + String details = ex.getMessage(); + if (details == null || details.isEmpty()) { + details = ex.toString(); + } + answers[0] = new Answer(cmd, false, details); + return _gson.toJson(answers); } final Answer[] answers = new Answer[1]; diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 8fe4c19a7e1..ead990b42b8 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -17,6 +17,7 @@ package com.cloud.vm; +import static com.cloud.configuration.ConfigurationManagerImpl.EXPOSE_ERRORS_TO_USER; import static com.cloud.configuration.ConfigurationManagerImpl.MIGRATE_VM_ACROSS_CLUSTERS; import java.lang.reflect.Field; @@ -49,7 +50,7 @@ import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; - +import com.cloud.hypervisor.KVMGuru; import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; import org.apache.cloudstack.annotation.AnnotationService; import org.apache.cloudstack.annotation.dao.AnnotationDao; @@ -309,7 +310,6 @@ import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; import com.google.gson.Gson; - public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMachineManager, VmWorkJobHandler, Listener, Configurable { public static final String VM_WORK_JOB_HANDLER = VirtualMachineManagerImpl.class.getSimpleName(); @@ -587,7 +587,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac Long deviceId = dataDiskDeviceIds.get(index++); String volumeName = deviceId == null ? "DATA-" + persistedVm.getId() : "DATA-" + persistedVm.getId() + "-" + String.valueOf(deviceId); volumeMgr.allocateRawVolume(Type.DATADISK, volumeName, dataDiskOfferingInfo.getDiskOffering(), dataDiskOfferingInfo.getSize(), - dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId); + dataDiskOfferingInfo.getMinIops(), dataDiskOfferingInfo.getMaxIops(), persistedVm, template, owner, deviceId, true); } } if (datadiskTemplateToDiskOfferingMap != null && !datadiskTemplateToDiskOfferingMap.isEmpty()) { @@ -597,7 +597,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac long diskOfferingSize = diskOffering.getDiskSize() / (1024 * 1024 * 1024); VMTemplateVO dataDiskTemplate = _templateDao.findById(dataDiskTemplateToDiskOfferingMap.getKey()); volumeMgr.allocateRawVolume(Type.DATADISK, "DATA-" + persistedVm.getId() + "-" + String.valueOf( diskNumber), diskOffering, diskOfferingSize, null, null, - persistedVm, dataDiskTemplate, owner, diskNumber); + persistedVm, dataDiskTemplate, owner, diskNumber, true); diskNumber++; } } @@ -627,7 +627,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac String rootVolumeName = String.format("ROOT-%s", vm.getId()); if (template.getFormat() == ImageFormat.ISO) { volumeMgr.allocateRawVolume(Type.ROOT, rootVolumeName, rootDiskOfferingInfo.getDiskOffering(), rootDiskOfferingInfo.getSize(), - rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null); + rootDiskOfferingInfo.getMinIops(), rootDiskOfferingInfo.getMaxIops(), vm, template, owner, null, true); } else if (Arrays.asList(ImageFormat.BAREMETAL, ImageFormat.EXTERNAL).contains(template.getFormat())) { logger.debug("{} has format [{}]. Skipping ROOT volume [{}] allocation.", template, template.getFormat(), rootVolumeName); } else { @@ -933,10 +933,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac public void start(final String vmUuid, final Map params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner) { try { advanceStart(vmUuid, params, planToDeploy, planner); - } catch (ConcurrentOperationException | InsufficientCapacityException e) { - throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } catch (ConcurrentOperationException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to concurrent operation.", vmUuid), e).add(VirtualMachine.class, vmUuid); + } catch (final InsufficientCapacityException e) { + final CallContext cctxt = CallContext.current(); + final Account account = cctxt.getCallingAccount(); + if (canExposeError(account)) { + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to [%s].", vmUuid, e.getMessage()), e).add(VirtualMachine.class, vmUuid); + } + throw new CloudRuntimeException(String.format("Unable to start a VM [%s] due to insufficient capacity.", vmUuid), e).add(VirtualMachine.class, vmUuid); } catch (final ResourceUnavailableException e) { - if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)){ + if (e.getScope() != null && e.getScope().equals(VirtualRouter.class)) { Account callingAccount = CallContext.current().getCallingAccount(); String errorSuffix = (callingAccount != null && callingAccount.getType() == Account.Type.ADMIN) ? String.format("Failure: %s", e.getMessage()) : @@ -1367,6 +1379,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType()); + Throwable lastKnownError = null; boolean canRetry = true; ExcludeList avoids = null; try { @@ -1390,7 +1403,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac int retry = StartRetry.value(); while (retry-- != 0) { - logger.debug("Instance start attempt #{}", (StartRetry.value() - retry)); + int attemptNumber = StartRetry.value() - retry; + logger.debug("Instance start attempt #{}", attemptNumber); if (reuseVolume) { final List vols = _volsDao.findReadyRootVolumesByInstance(vm.getId()); @@ -1456,8 +1470,13 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac reuseVolume = false; continue; } - throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(), - areAffinityGroupsAssociated(vmProfile)); + String message = String.format("Unable to create a deployment for %s after %s attempts", vmProfile, attemptNumber); + if (canExposeError(account) && lastKnownError != null) { + message += String.format(" Last known error: %s", lastKnownError.getMessage()); + throw new CloudRuntimeException(message, lastKnownError); + } else { + throw new InsufficientServerCapacityException(message, DataCenter.class, plan.getDataCenterId(), areAffinityGroupsAssociated(vmProfile)); + } } avoids.addHost(dest.getHost().getId()); @@ -1625,11 +1644,15 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying"); } } - logger.info("Unable to start VM on {} due to {}", dest.getHost(), (startAnswer == null ? " no start answer" : startAnswer.getDetails())); + String msg = String.format("Unable to start VM on %s due to %s", dest.getHost(), startAnswer == null ? "no start command answer" : startAnswer.getDetails()); + lastKnownError = new ExecutionException(msg); + if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) { + logger.error(msg, lastKnownError); break; } + logger.debug(msg, lastKnownError); } catch (OperationTimedoutException e) { logger.debug("Unable to send the start command to host {} failed to start VM: {}", dest.getHost(), vm); if (e.isActive()) { @@ -1639,6 +1662,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e); } catch (final ResourceUnavailableException e) { logger.warn("Unable to contact resource.", e); + lastKnownError = e; if (!avoids.add(e)) { if (e.getScope() == Volume.class || e.getScope() == Nic.class) { throw e; @@ -1695,10 +1719,22 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } if (startedVm == null) { - throw new CloudRuntimeException("Unable to start Instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details"); + String messageTmpl = "Unable to start Instance '%s' (%s)%s"; + String details; + if (canExposeError(account) && lastKnownError != null) { + details = ": " + lastKnownError.getMessage(); + } else { + details = ", see management server log for details"; + } + String message = String.format(messageTmpl, vm.getHostName(), vm.getUuid(), details); + throw new CloudRuntimeException(message, lastKnownError); } } + private boolean canExposeError(Account account) { + return (account != null && account.getType() == Account.Type.ADMIN) || Boolean.TRUE.equals(EXPOSE_ERRORS_TO_USER.value()); + } + protected void updateStartCommandWithExternalDetails(Host host, VirtualMachineTO vmTO, StartCommand command) { if (!HypervisorType.External.equals(host.getHypervisorType())) { return; @@ -2182,7 +2218,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) { final VirtualMachine vm = profile.getVirtualMachine(); Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); - StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup); updateStopCommandForExternalHypervisorType(vm.getHypervisorType(), profile, stpCmd); if (MapUtils.isNotEmpty(vlanToPersistenceMap)) { @@ -3983,19 +4018,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac } } - @Override - public boolean isVirtualMachineUpgradable(final VirtualMachine vm, final ServiceOffering offering) { - boolean isMachineUpgradable = true; - for (final HostAllocator allocator : hostAllocators) { - isMachineUpgradable = allocator.isVirtualMachineUpgradable(vm, offering); - if (!isMachineUpgradable) { - break; - } - } - - return isMachineUpgradable; - } - @Override public void reboot(final String vmUuid, final Map params) throws InsufficientCapacityException, ResourceUnavailableException { try { @@ -4435,11 +4457,6 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new InvalidParameterValueException("isSystem property is different for current service offering and new service offering"); } - if (!isVirtualMachineUpgradable(vmInstance, newServiceOffering)) { - throw new InvalidParameterValueException("Unable to upgrade virtual machine, not enough resources available " + "for an offering of " + - newServiceOffering.getCpu() + " cpu(s) at " + newServiceOffering.getSpeed() + " Mhz, and " + newServiceOffering.getRamSize() + " MB of memory"); - } - final List currentTags = StringUtils.csvTagsToList(currentDiskOffering.getTags()); final List newTags = StringUtils.csvTagsToList(newDiskOffering.getTags()); if (VolumeApiServiceImpl.MatchStoragePoolTagsWithDiskOffering.valueIn(vmInstance.getDataCenterId())) { @@ -5146,7 +5163,7 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac try { result = retrieveResultFromJobOutcomeAndThrowExceptionIfNeeded(outcome); } catch (Exception ex) { - throw new RuntimeException("Unhandled exception", ex); + throw new RuntimeException("Unable to reconfigure VM.", ex); } if (result != null) { @@ -5159,22 +5176,29 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private VMInstanceVO orchestrateReConfigureVm(String vmUuid, ServiceOffering oldServiceOffering, ServiceOffering newServiceOffering, boolean reconfiguringOnExistingHost) throws ResourceUnavailableException, ConcurrentOperationException { - final VMInstanceVO vm = _vmDao.findByUuid(vmUuid); + VMInstanceVO vm = _vmDao.findByUuid(vmUuid); HostVO hostVo = _hostDao.findById(vm.getHostId()); - Long clustedId = hostVo.getClusterId(); - Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clustedId); - Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clustedId); - boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clustedId); - boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clustedId); + Long clusterId = hostVo.getClusterId(); + Float memoryOvercommitRatio = CapacityManager.MemOverprovisioningFactor.valueIn(clusterId); + Float cpuOvercommitRatio = CapacityManager.CpuOverprovisioningFactor.valueIn(clusterId); + boolean divideMemoryByOverprovisioning = HypervisorGuruBase.VmMinMemoryEqualsMemoryDividedByMemOverprovisioningFactor.valueIn(clusterId); + boolean divideCpuByOverprovisioning = HypervisorGuruBase.VmMinCpuSpeedEqualsCpuSpeedDividedByCpuOverprovisioningFactor.valueIn(clusterId); int minMemory = (int)(newServiceOffering.getRamSize() / (divideMemoryByOverprovisioning ? memoryOvercommitRatio : 1)); int minSpeed = (int)(newServiceOffering.getSpeed() / (divideCpuByOverprovisioning ? cpuOvercommitRatio : 1)); - ScaleVmCommand scaleVmCommand = - new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, - newServiceOffering.getSpeed(), minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, newServiceOffering.getLimitCpuUse()); + Double cpuQuotaPercentage = null; + if (newServiceOffering.getLimitCpuUse() && vm.getHypervisorType().equals(HypervisorType.KVM)) { + KVMGuru kvmGuru = (KVMGuru) _hvGuruMgr.getGuru(vm.getHypervisorType()); + cpuQuotaPercentage = kvmGuru.getCpuQuotaPercentage(minSpeed, hostVo.getSpeed()); + } + + boolean limitCpuUseChange = oldServiceOffering.getLimitCpuUse() != newServiceOffering.getLimitCpuUse(); + ScaleVmCommand scaleVmCommand = new ScaleVmCommand(vm.getInstanceName(), newServiceOffering.getCpu(), minSpeed, newServiceOffering.getSpeed(), + minMemory * 1024L * 1024L, newServiceOffering.getRamSize() * 1024L * 1024L, + newServiceOffering.getLimitCpuUse(), cpuQuotaPercentage, limitCpuUseChange); scaleVmCommand.getVirtualMachine().setId(vm.getId()); scaleVmCommand.getVirtualMachine().setUuid(vm.getUuid()); @@ -5203,16 +5227,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac throw new CloudRuntimeException("Unable to scale vm due to " + (reconfigureAnswer == null ? "" : reconfigureAnswer.getDetails())); } - upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering); + if (reconfiguringOnExistingHost) { + _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); + } + + boolean vmUpgraded = upgradeVmDb(vm.getId(), newServiceOffering, oldServiceOffering); + if (vmUpgraded) { + vm = _vmDao.findById(vm.getId()); + } if (vm.getType().equals(VirtualMachine.Type.User)) { _userVmMgr.generateUsageEvent(vm, vm.isDisplayVm(), EventTypes.EVENT_VM_DYNAMIC_SCALE); } if (reconfiguringOnExistingHost) { - vm.setServiceOfferingId(oldServiceOffering.getId()); - _capacityMgr.releaseVmCapacity(vm, false, false, vm.getHostId()); - vm.setServiceOfferingId(newServiceOffering.getId()); _capacityMgr.allocateVmCapacity(vm, false); } @@ -5241,9 +5269,20 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac private void saveCustomOfferingDetails(long vmId, ServiceOffering serviceOffering) { Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vmId); - details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); - details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); - details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); + + // We need to restore only the customizable parameters. If we save a parameter that is not customizable and attempt + // to restore a VM snapshot, com.cloud.vm.UserVmManagerImpl.validateCustomParameters will fail. + ServiceOffering unfilledOffering = _serviceOfferingDao.findByIdIncludingRemoved(serviceOffering.getId()); + if (unfilledOffering.getCpu() == null) { + details.put(UsageEventVO.DynamicParameters.cpuNumber.name(), serviceOffering.getCpu().toString()); + } + if (unfilledOffering.getSpeed() == null) { + details.put(UsageEventVO.DynamicParameters.cpuSpeed.name(), serviceOffering.getSpeed().toString()); + } + if (unfilledOffering.getRamSize() == null) { + details.put(UsageEventVO.DynamicParameters.memory.name(), serviceOffering.getRamSize().toString()); + } + List detailList = new ArrayList<>(); for (Map.Entry entry: details.entrySet()) { VMInstanceDetailVO detailVO = new VMInstanceDetailVO(vmId, entry.getKey(), entry.getValue(), true); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 77ea965e50a..4262ee701aa 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -58,6 +58,7 @@ import org.apache.cloudstack.framework.messagebus.PublishScope; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.network.RoutedIpv4Manager; import org.apache.cloudstack.network.dao.NetworkPermissionDao; +import org.apache.cloudstack.reservation.dao.ReservationDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.ObjectUtils; @@ -86,6 +87,7 @@ import com.cloud.api.query.dao.DomainRouterJoinDao; import com.cloud.api.query.vo.DomainRouterJoinVO; import com.cloud.bgp.BGPService; import com.cloud.configuration.ConfigurationManager; +import com.cloud.configuration.Resource; import com.cloud.configuration.Resource.ResourceType; import com.cloud.dc.ASNumberVO; import com.cloud.dc.ClusterVO; @@ -214,6 +216,7 @@ import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.offerings.dao.NetworkOfferingDetailsDao; import com.cloud.offerings.dao.NetworkOfferingServiceMapDao; import com.cloud.resource.ResourceManager; +import com.cloud.resourcelimit.CheckedReservation; import com.cloud.server.ManagementServer; import com.cloud.user.Account; import com.cloud.user.ResourceLimitService; @@ -447,6 +450,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra ClusterDao clusterDao; @Inject RoutedIpv4Manager routedIpv4Manager; + @Inject + private ReservationDao reservationDao; protected StateMachine2 _stateMachine; ScheduledExecutorService _executor; @@ -858,6 +863,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra vpcId, offering.isRedundantRouter(), predefined.getExternalId()); vo.setDisplayNetwork(isDisplayNetworkEnabled == null || isDisplayNetworkEnabled); vo.setStrechedL2Network(offering.isSupportingStrechedL2()); + vo.setKeepMacAddressOnPublicNic(predefined.getKeepMacAddressOnPublicNic()); return vo; } @@ -2719,7 +2725,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, null, owner, null, pNtwk, pNtwk.getDataCenterId(), ACLType.Account, null, vpcId, null, null, true, null, null, null, true, null, null, - null, null, null, null, null, null); + null, null, null, null, null, null, true); } @Override @@ -2730,10 +2736,25 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, + networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, + isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, + ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize, true); + } + + @Override + @DB + public Network createGuestNetwork(final long networkOfferingId, final String name, final String displayText, final String gateway, final String cidr, String vlanId, + boolean bypassVlanOverlapCheck, String networkDomain, final Account owner, final Long domainId, final PhysicalNetwork pNtwk, + final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, + final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, + String routerIp, String routerIpv6, String ip4Dns1, String ip4Dns2, String ip6Dns1, String ip6Dns2, + Pair vrIfaceMTUs, Integer networkCidrSize, boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { // create Isolated/Shared/L2 network return createGuestNetwork(networkOfferingId, name, displayText, gateway, cidr, vlanId, bypassVlanOverlapCheck, networkDomain, owner, domainId, pNtwk, zoneId, aclType, subdomainAccess, vpcId, ip6Gateway, ip6Cidr, - isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize); + isDisplayNetworkEnabled, isolatedPvlan, isolatedPvlanType, externalId, false, routerIp, routerIpv6, + ip4Dns1, ip4Dns2, ip6Dns1, ip6Dns2, vrIfaceMTUs, networkCidrSize, keepMacAddressOnPublicNic); } @DB @@ -2742,7 +2763,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final long zoneId, final ACLType aclType, Boolean subdomainAccess, final Long vpcId, final String ip6Gateway, final String ip6Cidr, final Boolean isDisplayNetworkEnabled, final String isolatedPvlan, Network.PVlanType isolatedPvlanType, String externalId, final Boolean isPrivateNetwork, String routerIp, String routerIpv6, final String ip4Dns1, final String ip4Dns2, - final String ip6Dns1, final String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { + final String ip6Dns1, final String ip6Dns2, Pair vrIfaceMTUs, Integer networkCidrSize, + boolean keepMacAddressOnPublicNic) throws ConcurrentOperationException, InsufficientCapacityException, ResourceAllocationException { final NetworkOfferingVO ntwkOff = _networkOfferingDao.findById(networkOfferingId); final DataCenterVO zone = _dcDao.findById(zoneId); @@ -2752,12 +2774,6 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return null; } - final boolean updateResourceCount = resourceCountNeedsUpdate(ntwkOff, aclType); - //check resource limits - if (updateResourceCount) { - _resourceLimitMgr.checkResourceLimit(owner, ResourceType.network, isDisplayNetworkEnabled); - } - // Validate network offering if (ntwkOff.getState() != NetworkOffering.State.Enabled) { // see NetworkOfferingVO @@ -2776,218 +2792,219 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra boolean ipv6 = false; - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - ipv6 = true; - } - // Validate zone - if (zone.getNetworkType() == NetworkType.Basic) { - // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true - if (aclType == null || aclType != ACLType.Domain) { - throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + try (CheckedReservation networkReservation = new CheckedReservation(owner, domainId, Resource.ResourceType.network, null, null, 1L, reservationDao, _resourceLimitMgr)) { + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + ipv6 = true; } + // Validate zone + if (zone.getNetworkType() == NetworkType.Basic) { + // In Basic zone the network should have aclType=Domain, domainId=1, subdomainAccess=true + if (aclType == null || aclType != ACLType.Domain) { + throw new InvalidParameterValueException("Only AclType=Domain can be specified for network creation in Basic zone"); + } - // Only one guest network is supported in Basic zone - final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); - if (!guestNetworks.isEmpty()) { - throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); - } + // Only one guest network is supported in Basic zone + final List guestNetworks = _networksDao.listByZoneAndTrafficType(zone.getId(), TrafficType.Guest); + if (!guestNetworks.isEmpty()) { + throw new InvalidParameterValueException("Can't have more than one Guest network in zone with network type " + NetworkType.Basic); + } - // if zone is basic, only Shared network offerings w/o source nat service are allowed - if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { - throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " - + Service.SourceNat.getName() + " service are allowed"); - } + // if zone is basic, only Shared network offerings w/o source nat service are allowed + if (!(ntwkOff.getGuestType() == GuestType.Shared && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat))) { + throw new InvalidParameterValueException("For zone of type " + NetworkType.Basic + " only offerings of " + "guestType " + GuestType.Shared + " with disabled " + + Service.SourceNat.getName() + " service are allowed"); + } - if (domainId == null || domainId != Domain.ROOT_DOMAIN) { - throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); - } + if (domainId == null || domainId != Domain.ROOT_DOMAIN) { + throw new InvalidParameterValueException("Guest network in Basic zone should be dedicated to ROOT domain"); + } - if (subdomainAccess == null) { - subdomainAccess = true; - } else if (!subdomainAccess) { - throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); - } + if (subdomainAccess == null) { + subdomainAccess = true; + } else if (!subdomainAccess) { + throw new InvalidParameterValueException("Subdomain access should be set to true for the" + " guest network in the Basic zone"); + } - if (vlanId == null) { - vlanId = Vlan.UNTAGGED; - } else { - if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { - throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + if (vlanId == null) { + vlanId = Vlan.UNTAGGED; + } else { + if (!vlanId.equalsIgnoreCase(Vlan.UNTAGGED)) { + throw new InvalidParameterValueException("Only vlan " + Vlan.UNTAGGED + " can be created in " + "the zone of type " + NetworkType.Basic); + } + } + + } else if (zone.getNetworkType() == NetworkType.Advanced) { + if (zone.isSecurityGroupEnabled()) { + if (isolatedPvlan != null) { + throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); + } + // Only Account specific Isolated network with sourceNat service disabled are allowed in security group + // enabled zone + if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { + throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); + } + if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { + throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + } + } + + //don't allow eip/elb networks in Advance zone + if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { + throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); } } - } else if (zone.getNetworkType() == NetworkType.Advanced) { - if (zone.isSecurityGroupEnabled()) { - if (isolatedPvlan != null) { - throw new InvalidParameterValueException("Isolated Private VLAN is not supported with security group!"); - } - // Only Account specific Isolated network with sourceNat service disabled are allowed in security group - // enabled zone - if ((ntwkOff.getGuestType() != GuestType.Shared) && (ntwkOff.getGuestType() != GuestType.L2)) { - throw new InvalidParameterValueException("Only shared or L2 guest network can be created in security group enabled zone"); - } - if (_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat)) { - throw new InvalidParameterValueException("Service SourceNat is not allowed in security group enabled zone"); + if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { + _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); + } + + //TODO(VXLAN): Support VNI specified + // VlanId can be specified only when network offering supports it + final boolean vlanSpecified = vlanId != null; + if (vlanSpecified != ntwkOff.isSpecifyVlan()) { + if (vlanSpecified) { + if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + } + } else { + throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); } } - //don't allow eip/elb networks in Advance zone - if (ntwkOff.isElasticIp() || ntwkOff.isElasticLb()) { - throw new InvalidParameterValueException("Elastic IP and Elastic LB services are supported in zone of type " + NetworkType.Basic); - } - } - - if (ipv6 && !GuestType.Shared.equals(ntwkOff.getGuestType())) { - _networkModel.checkIp6CidrSizeEqualTo64(ip6Cidr); - } - - //TODO(VXLAN): Support VNI specified - // VlanId can be specified only when network offering supports it - final boolean vlanSpecified = vlanId != null; - if (vlanSpecified != ntwkOff.isSpecifyVlan()) { if (vlanSpecified) { - if (!isSharedNetworkWithoutSpecifyVlan(ntwkOff) && !isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - throw new InvalidParameterValueException("Can't specify vlan; corresponding offering says specifyVlan=false"); + URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); + // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks + URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; + if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { + bypassVlanOverlapCheck = true; } - } else { - throw new InvalidParameterValueException("Vlan has to be specified; corresponding offering says specifyVlan=true"); - } - } - - if (vlanSpecified) { - URI uri = encodeVlanIdIntoBroadcastUri(vlanId, pNtwk); - // Aux: generate secondary URI for secondary VLAN ID (if provided) for performing checks - URI secondaryUri = StringUtils.isNotBlank(isolatedPvlan) ? BroadcastDomainType.fromString(isolatedPvlan) : null; - if (isSharedNetworkWithoutSpecifyVlan(ntwkOff) || isPrivateGatewayWithoutSpecifyVlan(ntwkOff)) { - bypassVlanOverlapCheck = true; - } - //don't allow to specify vlan tag used by physical network for dynamic vlan allocation - if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) - && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { - throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " - + zone.getName()); - } - if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && - _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { - throw new InvalidParameterValueException(String.format( - "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", - isolatedPvlan, zone)); - } - if (!UuidUtils.isUuid(vlanId)) { - // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone - if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { - if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - vlanId, zone)); - } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network vlans in zone %s", - isolatedPvlan, zone)); - } else { - final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); - //for the network that is created as part of private gateway, - //the vnet is not coming from the data center vnet table, so the list can be empty - if (!dcVnets.isEmpty()) { - final DataCenterVnetVO dcVnet = dcVnets.get(0); - // Fail network creation if specified vlan is dedicated to a different account - if (dcVnet.getAccountGuestVlanMapId() != null) { - final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); - final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); - if (map.getAccountId() != owner.getAccountId()) { - throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); - } - // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool - } else { - final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); - if (maps != null && !maps.isEmpty()) { - final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); - final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); - if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { - throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " - + owner.getAccountName()); + //don't allow to specify vlan tag used by physical network for dynamic vlan allocation + if (!(bypassVlanOverlapCheck && (ntwkOff.getGuestType() == GuestType.Shared || isPrivateNetwork)) + && _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(uri)).size() > 0) { + throw new InvalidParameterValueException("The VLAN tag to use for new guest network, " + vlanId + " is already being used for dynamic vlan allocation for the guest network in zone " + + zone.getName()); + } + if (secondaryUri != null && !(bypassVlanOverlapCheck && ntwkOff.getGuestType() == GuestType.Shared) && + _dcDao.findVnet(zoneId, pNtwk.getId(), BroadcastDomainType.getValue(secondaryUri)).size() > 0) { + throw new InvalidParameterValueException(String.format( + "The VLAN tag for isolated PVLAN %s is already being used for dynamic vlan allocation for the guest network in zone %s", + isolatedPvlan, zone)); + } + if (!UuidUtils.isUuid(vlanId)) { + // For Isolated and L2 networks, don't allow to create network with vlan that already exists in the zone + if (!hasGuestBypassVlanOverlapCheck(bypassVlanOverlapCheck, ntwkOff, isPrivateNetwork)) { + if (_networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + vlanId, zone)); + } else if (secondaryUri != null && _networksDao.listByZoneAndUriAndGuestType(zoneId, secondaryUri.toString(), null).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network vlans in zone %s", + isolatedPvlan, zone)); + } else { + final List dcVnets = _datacenterVnetDao.findVnet(zoneId, BroadcastDomainType.getValue(uri)); + //for the network that is created as part of private gateway, + //the vnet is not coming from the data center vnet table, so the list can be empty + if (!dcVnets.isEmpty()) { + final DataCenterVnetVO dcVnet = dcVnets.get(0); + // Fail network creation if specified vlan is dedicated to a different account + if (dcVnet.getAccountGuestVlanMapId() != null) { + final Long accountGuestVlanMapId = dcVnet.getAccountGuestVlanMapId(); + final AccountGuestVlanMapVO map = _accountGuestVlanMapDao.findById(accountGuestVlanMapId); + if (map.getAccountId() != owner.getAccountId()) { + throw new InvalidParameterValueException("Vlan " + vlanId + " is dedicated to a different account"); + } + // Fail network creation if owner has a dedicated range of vlans but the specified vlan belongs to the system pool + } else { + final List maps = _accountGuestVlanMapDao.listAccountGuestVlanMapsByAccount(owner.getAccountId()); + if (maps != null && !maps.isEmpty()) { + final int vnetsAllocatedToAccount = _datacenterVnetDao.countVnetsAllocatedToAccount(zoneId, owner.getAccountId()); + final int vnetsDedicatedToAccount = _datacenterVnetDao.countVnetsDedicatedToAccount(zoneId, owner.getAccountId()); + if (vnetsAllocatedToAccount < vnetsDedicatedToAccount) { + throw new InvalidParameterValueException("Specified vlan " + vlanId + " doesn't belong" + " to the vlan range dedicated to the owner " + + owner.getAccountName()); + } } } } } + } else { + // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or + // shared network with same Vlan ID in the zone + if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + throw new InvalidParameterValueException(String.format( + "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + } } - } else { - // don't allow to creating shared network with given Vlan ID, if there already exists a isolated network or - // shared network with same Vlan ID in the zone - if (!bypassVlanOverlapCheck && _networksDao.listByZoneAndUriAndGuestType(zoneId, uri.toString(), GuestType.Isolated).size() > 0) { + } + + } + + // If networkDomain is not specified, take it from the global configuration + if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { + final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), + Service.Dns); + final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); + if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { + if (networkDomain != null) { + // TBD: NetworkOfferingId and zoneId. Send uuids instead. throw new InvalidParameterValueException(String.format( - "There is an existing isolated/shared network that overlaps with vlan id:%s in zone %s", vlanId, zone)); + "Domain name change is not supported by network offering id=%d in zone %s", + networkOfferingId, zone)); } - } - } - - } - - // If networkDomain is not specified, take it from the global configuration - if (_networkModel.areServicesSupportedByNetworkOffering(networkOfferingId, Service.Dns)) { - final Map dnsCapabilities = _networkModel.getNetworkOfferingServiceCapabilities(_entityMgr.findById(NetworkOffering.class, networkOfferingId), - Service.Dns); - final String isUpdateDnsSupported = dnsCapabilities.get(Capability.AllowDnsSuffixModification); - if (isUpdateDnsSupported == null || !Boolean.valueOf(isUpdateDnsSupported)) { - if (networkDomain != null) { - // TBD: NetworkOfferingId and zoneId. Send uuids instead. - throw new InvalidParameterValueException(String.format( - "Domain name change is not supported by network offering id=%d in zone %s", - networkOfferingId, zone)); - } - } else { - if (networkDomain == null) { - // 1) Get networkDomain from the corresponding account/domain/zone - if (aclType == ACLType.Domain) { - networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); - } else if (aclType == ACLType.Account) { - networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); - } - - // 2) If null, generate networkDomain using domain suffix from the global config variables - if (networkDomain == null) { - networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); - } - } else { - // validate network domain - if (!NetUtils.verifyDomainName(networkDomain)) { - throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " - + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " - + "and the hyphen ('-'); can't start or end with \"-\""); + if (networkDomain == null) { + // 1) Get networkDomain from the corresponding account/domain/zone + if (aclType == ACLType.Domain) { + networkDomain = _networkModel.getDomainNetworkDomain(domainId, zoneId); + } else if (aclType == ACLType.Account) { + networkDomain = _networkModel.getAccountNetworkDomain(owner.getId(), zoneId); + } + + // 2) If null, generate networkDomain using domain suffix from the global config variables + if (networkDomain == null) { + networkDomain = "cs" + Long.toHexString(owner.getId()) + GuestDomainSuffix.valueIn(zoneId); + } + + } else { + // validate network domain + if (!NetUtils.verifyDomainName(networkDomain)) { + throw new InvalidParameterValueException("Invalid network domain. Total length shouldn't exceed 190 chars. Each domain " + + "label must be between 1 and 63 characters long, can contain ASCII letters 'a' through 'z', the digits '0' through '9', " + + "and the hyphen ('-'); can't start or end with \"-\""); + } } } } - } - // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x - // limitation, remove after we introduce support for multiple ip ranges - // with different Cidrs for the same Shared network - final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced - && ntwkOff.getTrafficType() == TrafficType.Guest - && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) - && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); - if (cidr == null && ip6Cidr == null && cidrRequired) { - if (ntwkOff.getGuestType() == GuestType.Shared) { - throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); - } else { - throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + // In Advance zone Cidr for Shared networks and Isolated networks w/o source nat service can't be NULL - 2.2.x + // limitation, remove after we introduce support for multiple ip ranges + // with different Cidrs for the same Shared network + final boolean cidrRequired = zone.getNetworkType() == NetworkType.Advanced + && ntwkOff.getTrafficType() == TrafficType.Guest + && (ntwkOff.getGuestType() == GuestType.Shared || (ntwkOff.getGuestType() == GuestType.Isolated + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.SourceNat) + && !_networkModel.areServicesSupportedByNetworkOffering(ntwkOff.getId(), Service.Gateway))); + if (cidr == null && ip6Cidr == null && cidrRequired) { + if (ntwkOff.getGuestType() == GuestType.Shared) { + throw new InvalidParameterValueException(String.format("Gateway/netmask are required when creating %s networks.", Network.GuestType.Shared)); + } else { + throw new InvalidParameterValueException("gateway/netmask are required when create network of" + " type " + GuestType.Isolated + " with service " + Service.SourceNat.getName() + " disabled"); + } } - } - checkL2OfferingServices(ntwkOff); + checkL2OfferingServices(ntwkOff); - // No cidr can be specified in Basic zone - if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { - throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); - } + // No cidr can be specified in Basic zone + if (zone.getNetworkType() == NetworkType.Basic && cidr != null) { + throw new InvalidParameterValueException("StartIp/endIp/gateway/netmask can't be specified for zone of type " + NetworkType.Basic); + } - // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 - if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && - !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { + // Check if cidr is RFC1918 compliant if the network is Guest Isolated for IPv4 + if (cidr != null && (ntwkOff.getGuestType() == Network.GuestType.Isolated && ntwkOff.getTrafficType() == TrafficType.Guest) && + !NetUtils.validateGuestCidr(cidr, !ConfigurationManager.AllowNonRFC1918CompliantIPs.value())) { throw new InvalidParameterValueException("Virtual Guest Cidr " + cidr + " is not RFC 1918 or 6598 compliant"); - } + } final String networkDomainFinal = networkDomain; final String vlanIdFinal = vlanId; @@ -3003,75 +3020,75 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra final NetworkVO userNetwork = new NetworkVO(); userNetwork.setNetworkDomain(networkDomainFinal); - if (cidr != null && gateway != null) { - userNetwork.setCidr(cidr); - userNetwork.setGateway(gateway); - } + if (cidr != null && gateway != null) { + userNetwork.setCidr(cidr); + userNetwork.setGateway(gateway); + } - if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { - userNetwork.setIp6Cidr(ip6Cidr); - userNetwork.setIp6Gateway(ip6Gateway); - } + if (StringUtils.isNoneBlank(ip6Gateway, ip6Cidr)) { + userNetwork.setIp6Cidr(ip6Cidr); + userNetwork.setIp6Gateway(ip6Gateway); + } - if (externalId != null) { - userNetwork.setExternalId(externalId); - } + if (externalId != null) { + userNetwork.setExternalId(externalId); + } - if (StringUtils.isNotBlank(routerIp)) { - userNetwork.setRouterIp(routerIp); - } + if (StringUtils.isNotBlank(routerIp)) { + userNetwork.setRouterIp(routerIp); + } - if (StringUtils.isNotBlank(routerIpv6)) { - userNetwork.setRouterIpv6(routerIpv6); - } + if (StringUtils.isNotBlank(routerIpv6)) { + userNetwork.setRouterIpv6(routerIpv6); + } - if (vrIfaceMTUs != null) { - if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { - userNetwork.setPublicMtu(vrIfaceMTUs.first()); + if (vrIfaceMTUs != null) { + if (vrIfaceMTUs.first() != null && vrIfaceMTUs.first() > 0) { + userNetwork.setPublicMtu(vrIfaceMTUs.first()); + } else { + userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); + } + + if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { + userNetwork.setPrivateMtu(vrIfaceMTUs.second()); + } else { + userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); + } } else { userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - } - - if (vrIfaceMTUs.second() != null && vrIfaceMTUs.second() > 0) { - userNetwork.setPrivateMtu(vrIfaceMTUs.second()); - } else { userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); } - } else { - userNetwork.setPublicMtu(Integer.valueOf(NetworkService.VRPublicInterfaceMtu.defaultValue())); - userNetwork.setPrivateMtu(Integer.valueOf(NetworkService.VRPrivateInterfaceMtu.defaultValue())); - } - if (!GuestType.L2.equals(userNetwork.getGuestType())) { - if (StringUtils.isNotBlank(ip4Dns1)) { - userNetwork.setDns1(ip4Dns1); - } - if (StringUtils.isNotBlank(ip4Dns2)) { - userNetwork.setDns2(ip4Dns2); - } - if (StringUtils.isNotBlank(ip6Dns1)) { - userNetwork.setIp6Dns1(ip6Dns1); - } - if (StringUtils.isNotBlank(ip6Dns2)) { - userNetwork.setIp6Dns2(ip6Dns2); - } - } - - if (vlanIdFinal != null) { - if (isolatedPvlan == null) { - URI uri = null; - if (UuidUtils.isUuid(vlanIdFinal)) { - //Logical router's UUID provided as VLAN_ID - userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field - } else { - uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + if (!GuestType.L2.equals(userNetwork.getGuestType())) { + if (StringUtils.isNotBlank(ip4Dns1)) { + userNetwork.setDns1(ip4Dns1); } - - if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { - throw new InvalidParameterValueException(String.format( - "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", - vlanIdFinal, zone)); + if (StringUtils.isNotBlank(ip4Dns2)) { + userNetwork.setDns2(ip4Dns2); } + if (StringUtils.isNotBlank(ip6Dns1)) { + userNetwork.setIp6Dns1(ip6Dns1); + } + if (StringUtils.isNotBlank(ip6Dns2)) { + userNetwork.setIp6Dns2(ip6Dns2); + } + } + + if (vlanIdFinal != null) { + if (isolatedPvlan == null) { + URI uri = null; + if (UuidUtils.isUuid(vlanIdFinal)) { + //Logical router's UUID provided as VLAN_ID + userNetwork.setVlanIdAsUUID(vlanIdFinal); //Set transient field + } else { + uri = encodeVlanIdIntoBroadcastUri(vlanIdFinal, pNtwk); + } + + if (_networksDao.listByPhysicalNetworkPvlan(physicalNetworkId, uri.toString()).size() > 0) { + throw new InvalidParameterValueException(String.format( + "Network with vlan %s already exists or overlaps with other network pvlans in zone %s", + vlanIdFinal, zone)); + } userNetwork.setBroadcastUri(uri); if (!vlanIdFinal.equalsIgnoreCase(Vlan.UNTAGGED)) { @@ -3095,6 +3112,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } userNetwork.setNetworkCidrSize(networkCidrSize); + userNetwork.setKeepMacAddressOnPublicNic(keepMacAddressOnPublicNic); final List networks = setupNetwork(owner, ntwkOff, userNetwork, plan, name, displayText, true, domainId, aclType, subdomainAccessFinal, vpcId, isDisplayNetworkEnabled); Network network; @@ -3115,8 +3133,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } } - if (updateResourceCount) { - _resourceLimitMgr.incrementResourceCount(owner.getId(), ResourceType.network, isDisplayNetworkEnabled); + if (isResourceCountUpdateNeeded(ntwkOff)) { + changeAccountResourceCountOrRecalculateDomainResourceCount(owner.getAccountId(), domainId, isDisplayNetworkEnabled, true); } UsageEventUtils.publishNetworkCreation(network); @@ -3127,6 +3145,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra CallContext.current().setEventDetails("Network ID: " + network.getUuid()); CallContext.current().putContextParameter(Network.class, network.getUuid()); return network; + } } @Override @@ -3492,9 +3511,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } final NetworkOffering ntwkOff = _entityMgr.findById(NetworkOffering.class, networkFinal.getNetworkOfferingId()); - final boolean updateResourceCount = resourceCountNeedsUpdate(ntwkOff, networkFinal.getAclType()); - if (updateResourceCount) { - _resourceLimitMgr.decrementResourceCount(networkFinal.getAccountId(), ResourceType.network, networkFinal.getDisplayNetwork()); + if (isResourceCountUpdateNeeded(ntwkOff)) { + changeAccountResourceCountOrRecalculateDomainResourceCount(networkFinal.getAccountId(), networkFinal.getDomainId(), networkFinal.getDisplayNetwork(), false); } } return deletedVlans.second(); @@ -3517,6 +3535,23 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return success; } + /** + * If it is a shared network with {@link ACLType#Domain}, it will belong to account {@link Account#ACCOUNT_ID_SYSTEM} and the resources will be not incremented for the + * domain. Therefore, we force the recalculation of the domain's resource count in this case. Otherwise, it will change the count for the account owner. + * @param incrementAccountResourceCount If true, the account resource count will be incremented by 1; otherwise, it will decremented by 1. + */ + private void changeAccountResourceCountOrRecalculateDomainResourceCount(Long accountId, Long domainId, boolean displayNetwork, boolean incrementAccountResourceCount) { + if (Account.ACCOUNT_ID_SYSTEM == accountId && ObjectUtils.isNotEmpty(domainId)) { + _resourceLimitMgr.recalculateDomainResourceCount(domainId, ResourceType.network, null); + } else { + if (incrementAccountResourceCount) { + _resourceLimitMgr.incrementResourceCount(accountId, ResourceType.network, displayNetwork); + } else { + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.network, displayNetwork); + } + } + } + private void publishDeletedVlanRanges(List deletedVlanRangeToPublish) { if (CollectionUtils.isNotEmpty(deletedVlanRangeToPublish)) { for (VlanVO vlan : deletedVlanRangeToPublish) { @@ -3526,10 +3561,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra } @Override - public boolean resourceCountNeedsUpdate(final NetworkOffering ntwkOff, final ACLType aclType) { - //Update resource count only for Isolated account specific non-system networks - final boolean updateResourceCount = ntwkOff.getGuestType() == GuestType.Isolated && !ntwkOff.isSystemOnly() && aclType == ACLType.Account; - return updateResourceCount; + public boolean isResourceCountUpdateNeeded(NetworkOffering networkOffering) { + return !networkOffering.isSystemOnly(); } protected Pair> deleteVlansInNetwork(final NetworkVO network, final long userId, final Account callerAccount) { @@ -4924,6 +4957,7 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra return new ConfigKey[]{NetworkGcWait, NetworkGcInterval, NetworkLockTimeout, DeniedRoutes, GuestDomainSuffix, NetworkThrottlingRate, MinVRVersion, PromiscuousMode, MacAddressChanges, ForgedTransmits, MacLearning, RollingRestartEnabled, - TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN}; + TUNGSTEN_ENABLED, NSX_ENABLED, NETRIS_ENABLED, NETWORK_LB_HAPROXY_MAX_CONN, + NETWORK_LB_HAPROXY_IDLE_TIMEOUT}; } } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index e8c75afa81c..bf3985d3ce7 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -40,6 +40,7 @@ import javax.naming.ConfigurationException; import com.cloud.deploy.DeploymentClusterPlanner; import com.cloud.exception.ResourceAllocationException; +import com.cloud.resourcelimit.ReservationHelper; import com.cloud.storage.DiskOfferingVO; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.dao.VMTemplateDao; @@ -82,6 +83,7 @@ import org.apache.cloudstack.framework.jobs.AsyncJobManager; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.resourcedetail.DiskOfferingDetailVO; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; +import org.apache.cloudstack.resourcelimit.Reserver; import org.apache.cloudstack.secret.PassphraseVO; import org.apache.cloudstack.secret.dao.PassphraseDao; import org.apache.cloudstack.snapshot.SnapshotHelper; @@ -862,7 +864,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati @ActionEvent(eventType = EventTypes.EVENT_VOLUME_CREATE, eventDescription = "creating volume", create = true) @Override public DiskProfile allocateRawVolume(Type type, String name, DiskOffering offering, Long size, Long minIops, Long maxIops, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId) { + Long deviceId, boolean incrementResourceCount) { if (size == null) { size = offering.getDiskSize(); } else { @@ -901,7 +903,7 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati saveVolumeDetails(offering.getId(), vol.getId()); // Save usage event and update resource count for user vm volumes - if (vm.getType() == VirtualMachine.Type.User) { + if (vm.getType() == VirtualMachine.Type.User && incrementResourceCount) { UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, vol.getAccountId(), vol.getDataCenterId(), vol.getId(), vol.getName(), offering.getId(), null, size, Volume.class.getName(), vol.getUuid(), vol.getInstanceId(), vol.isDisplayVolume()); _resourceLimitMgr.incrementVolumeResourceCount(vm.getAccountId(), vol.isDisplayVolume(), vol.getSize(), offering); @@ -1938,14 +1940,20 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati template == null ? null : template.getSize(), vol.getPassphraseId() != null); - if (newSize != vol.getSize()) { - DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + if (newSize == vol.getSize()) { + return; + } + + DiskOfferingVO diskOffering = diskOfferingDao.findByIdIncludingRemoved(vol.getDiskOfferingId()); + + List reservations = new ArrayList<>(); + try { VMInstanceVO vm = vol.getInstanceId() != null ? vmInstanceDao.findById(vol.getInstanceId()) : null; if (vm == null || vm.getType() == VirtualMachine.Type.User) { // Update resource count for user vm volumes when volume is attached if (newSize > vol.getSize()) { _resourceLimitMgr.checkPrimaryStorageResourceLimit(_accountMgr.getActiveAccountById(vol.getAccountId()), - vol.isDisplay(), newSize - vol.getSize(), diskOffering); + vol.isDisplay(), newSize - vol.getSize(), diskOffering, reservations); _resourceLimitMgr.incrementVolumePrimaryStorageResourceCount(vol.getAccountId(), vol.isDisplay(), newSize - vol.getSize(), diskOffering); } else { @@ -1953,9 +1961,11 @@ public class VolumeOrchestrator extends ManagerBase implements VolumeOrchestrati vol.getSize() - newSize, diskOffering); } } - vol.setSize(newSize); - _volsDao.persist(vol); + } finally { + ReservationHelper.closeAll(reservations); } + vol.setSize(newSize); + _volsDao.persist(vol); } @Override diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java index 01afd0780f7..e4047cf7973 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDao.java @@ -27,6 +27,6 @@ public interface AccountVlanMapDao extends GenericDao { public List listAccountVlanMapsByVlan(long vlanDbId); - public AccountVlanMapVO findAccountVlanMap(long accountId, long vlanDbId); + public AccountVlanMapVO findAccountVlanMap(Long accountId, long vlanDbId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java index 12114770f11..0844bb77caa 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/AccountVlanMapDaoImpl.java @@ -48,9 +48,9 @@ public class AccountVlanMapDaoImpl extends GenericDaoBase sc = AccountVlanSearch.create(); - sc.setParameters("accountId", accountId); + sc.setParametersIfNotNull("accountId", accountId); sc.setParameters("vlanDbId", vlanDbId); return findOneIncludingRemovedBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java index 6af16bbace9..d14ccbe86ca 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDao.java @@ -24,5 +24,5 @@ import com.cloud.utils.db.GenericDao; public interface DomainVlanMapDao extends GenericDao { public List listDomainVlanMapsByDomain(long domainId); public List listDomainVlanMapsByVlan(long vlanDbId); - public DomainVlanMapVO findDomainVlanMap(long domainId, long vlanDbId); + public DomainVlanMapVO findDomainVlanMap(Long domainId, long vlanDbId); } diff --git a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java index f789721d5fd..0b4c781349f 100644 --- a/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/dc/dao/DomainVlanMapDaoImpl.java @@ -46,9 +46,9 @@ public class DomainVlanMapDaoImpl extends GenericDaoBase } @Override - public DomainVlanMapVO findDomainVlanMap(long domainId, long vlanDbId) { + public DomainVlanMapVO findDomainVlanMap(Long domainId, long vlanDbId) { SearchCriteria sc = DomainVlanSearch.create(); - sc.setParameters("domainId", domainId); + sc.setParametersIfNotNull("domainId", domainId); sc.setParameters("vlanDbId", vlanDbId); return findOneIncludingRemovedBy(sc); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java index 090b019334f..5f5b2affee0 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDao.java @@ -218,7 +218,7 @@ public interface HostDao extends GenericDao, StateDao listOrderedHostsHypervisorVersionsInDatacenter(long datacenterId, HypervisorType hypervisorType); - List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags); + List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags); List findClustersThatMatchHostTagRule(String computeOfferingTags); diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java index 2d8fcca6cdb..99c9a979c3b 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostDaoImpl.java @@ -1520,7 +1520,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao } } - public List findHostsWithTagRuleThatMatchComputeOferringTags(String computeOfferingTags) { + public List findHostsWithTagRuleThatMatchComputeOfferingTags(String computeOfferingTags) { List hostTagVOList = _hostTagsDao.findHostRuleTags(); List result = new ArrayList<>(); for (HostTagVO rule: hostTagVOList) { @@ -1534,7 +1534,7 @@ public class HostDaoImpl extends GenericDaoBase implements HostDao public List findClustersThatMatchHostTagRule(String computeOfferingTags) { Set result = new HashSet<>(); - List hosts = findHostsWithTagRuleThatMatchComputeOferringTags(computeOfferingTags); + List hosts = findHostsWithTagRuleThatMatchComputeOfferingTags(computeOfferingTags); for (HostVO host: hosts) { result.add(host.getClusterId()); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java index 7a00829fd44..0d86ca0e48c 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java @@ -45,4 +45,9 @@ public interface HostTagsDao extends GenericDao { HostTagResponse newHostTagResponse(HostTagVO hostTag); List searchByIds(Long... hostTagIds); + + /** + * List all host tags defined on hosts within a cluster + */ + List listByClusterId(Long clusterId); } diff --git a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java index 4aa14a31cfc..d3fee6a2676 100644 --- a/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.response.HostTagResponse; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -43,9 +44,12 @@ public class HostTagsDaoImpl extends GenericDaoBase implements private final SearchBuilder stSearch; private final SearchBuilder tagIdsearch; private final SearchBuilder ImplicitTagsSearch; + private final GenericSearchBuilder tagSearch; @Inject private ConfigurationDao _configDao; + @Inject + private HostDao hostDao; public HostTagsDaoImpl() { HostSearch = createSearchBuilder(); @@ -72,6 +76,11 @@ public class HostTagsDaoImpl extends GenericDaoBase implements ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ); ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ); ImplicitTagsSearch.done(); + + tagSearch = createSearchBuilder(String.class); + tagSearch.selectFields(tagSearch.entity().getTag()); + tagSearch.and("hostIdIN", tagSearch.entity().getHostId(), SearchCriteria.Op.IN); + tagSearch.done(); } @Override @@ -235,4 +244,15 @@ public class HostTagsDaoImpl extends GenericDaoBase implements return tagList; } + + @Override + public List listByClusterId(Long clusterId) { + List hostIds = hostDao.listIdsByClusterId(clusterId); + if (CollectionUtils.isEmpty(hostIds)) { + return new ArrayList<>(); + } + SearchCriteria sc = tagSearch.create(); + sc.setParameters("hostIdIN", hostIds.toArray()); + return customSearch(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java index 4e8b6204f72..9f7ffabac93 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkDaoImpl.java @@ -193,6 +193,7 @@ public class NetworkDaoImpl extends GenericDaoBaseimplements Ne PersistentNetworkSearch.and("id", PersistentNetworkSearch.entity().getId(), Op.NEQ); PersistentNetworkSearch.and("guestType", PersistentNetworkSearch.entity().getGuestType(), Op.IN); PersistentNetworkSearch.and("broadcastUri", PersistentNetworkSearch.entity().getBroadcastUri(), Op.EQ); + PersistentNetworkSearch.and("dc", PersistentNetworkSearch.entity().getDataCenterId(), Op.EQ); PersistentNetworkSearch.and("removed", PersistentNetworkSearch.entity().getRemoved(), Op.NULL); final SearchBuilder persistentNtwkOffJoin = _ntwkOffDao.createSearchBuilder(); persistentNtwkOffJoin.and("persistent", persistentNtwkOffJoin.entity().isPersistent(), Op.EQ); diff --git a/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java b/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java index 02abaacd854..f2572ba91c2 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/NetworkVO.java @@ -203,9 +203,13 @@ public class NetworkVO implements Network { @Column(name = "private_mtu") Integer privateMtu; + @Column(name = "keep_mac_address_on_public_nic") + private boolean keepMacAddressOnPublicNic = true; + @Transient Integer networkCidrSize; + public NetworkVO() { uuid = UUID.randomUUID().toString(); } @@ -773,4 +777,13 @@ public class NetworkVO implements Network { public void setNetworkCidrSize(Integer networkCidrSize) { this.networkCidrSize = networkCidrSize; } + + @Override + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + + public void setKeepMacAddressOnPublicNic(boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java index ccba6bb1889..606bdaaaa7a 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDao.java @@ -19,9 +19,21 @@ package com.cloud.network.dao; import com.cloud.network.vo.PublicIpQuarantineVO; import com.cloud.utils.db.GenericDao; +import java.util.Date; +import java.util.List; + public interface PublicIpQuarantineDao extends GenericDao { PublicIpQuarantineVO findByPublicIpAddressId(long publicIpAddressId); PublicIpQuarantineVO findByIpAddress(String publicIpAddress); + + /** + * Returns a list of public IP addresses that are actively quarantined at the specified date and the previous owner differs from the specified user. + * + * @param userId used to check against the IP's previous owner; + * @param date used to check if the quarantine is active; + * @return a list of PublicIpQuarantineVOs. + */ + List listQuarantinedIpAddressesToUser(Long userId, Date date); } diff --git a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java index a1b789b8a46..0c47a0d36e3 100644 --- a/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/network/dao/PublicIpQuarantineDaoImpl.java @@ -26,6 +26,8 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.inject.Inject; +import java.util.Date; +import java.util.List; @Component public class PublicIpQuarantineDaoImpl extends GenericDaoBase implements PublicIpQuarantineDao { @@ -33,6 +35,8 @@ public class PublicIpQuarantineDaoImpl extends GenericDaoBase ipAddressSearchBuilder; + private SearchBuilder quarantinedIpAddressesSearch; + @Inject IPAddressDao ipAddressDao; @@ -47,8 +51,16 @@ public class PublicIpQuarantineDaoImpl extends GenericDaoBase listQuarantinedIpAddressesToUser(Long userId, Date date) { + SearchCriteria sc = quarantinedIpAddressesSearch.create(); + + sc.setParameters("previousOwnerId", userId); + sc.setParameters("endDate", date); + + return searchIncludingRemoved(sc, null, false, false); + } } diff --git a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java index e942eadb8ff..742d3f2f82e 100644 --- a/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java +++ b/engine/schema/src/main/java/com/cloud/network/vpc/VpcVO.java @@ -108,6 +108,9 @@ public class VpcVO implements Vpc { @Column(name = "use_router_ip_resolver") boolean useRouterIpResolver = false; + @Column(name = "keep_mac_address_on_public_nic") + private boolean keepMacAddressOnPublicNic = true; + @Transient boolean rollingRestart = false; @@ -321,4 +324,13 @@ public class VpcVO implements Vpc { public void setUseRouterIpResolver(boolean useRouterIpResolver) { this.useRouterIpResolver = useRouterIpResolver; } + + @Override + public boolean getKeepMacAddressOnPublicNic() { + return keepMacAddressOnPublicNic; + } + + public void setKeepMacAddressOnPublicNic(boolean keepMacAddressOnPublicNic) { + this.keepMacAddressOnPublicNic = keepMacAddressOnPublicNic; + } } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java index 80e1b7d4d4b..f167b573187 100755 --- a/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/SnapshotDaoImpl.java @@ -170,6 +170,7 @@ public class SnapshotDaoImpl extends GenericDaoBase implements CountSnapshotsByAccount.select(null, Func.COUNT, null); CountSnapshotsByAccount.and("account", CountSnapshotsByAccount.entity().getAccountId(), SearchCriteria.Op.EQ); CountSnapshotsByAccount.and("status", CountSnapshotsByAccount.entity().getState(), SearchCriteria.Op.NIN); + CountSnapshotsByAccount.and("snapshotTypeNEQ", CountSnapshotsByAccount.entity().getSnapshotType(), SearchCriteria.Op.NIN); CountSnapshotsByAccount.and("removed", CountSnapshotsByAccount.entity().getRemoved(), SearchCriteria.Op.NULL); CountSnapshotsByAccount.done(); @@ -220,6 +221,7 @@ public class SnapshotDaoImpl extends GenericDaoBase implements SearchCriteria sc = CountSnapshotsByAccount.create(); sc.setParameters("account", accountId); sc.setParameters("status", State.Error, State.Destroyed); + sc.setParameters("snapshotTypeNEQ", Snapshot.Type.GROUP.ordinal()); return customSearch(sc, null).get(0); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java index 445a59310fb..377b2f91375 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseVersionHierarchy.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.upgrade; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -96,7 +97,9 @@ public final class DatabaseVersionHierarchy { // we cannot find the version specified, so get the // most recent one immediately before this version if (!contains(fromVersion)) { - return getPath(getRecentVersion(fromVersion), toVersion); + DbUpgrade[] dbUpgrades = getPath(getRecentVersion(fromVersion), toVersion); + return Arrays.stream(dbUpgrades).filter(up -> CloudStackVersion.compare(up.getUpgradedVersion(), fromVersion.toString()) > 0) + .toArray(DbUpgrade[]::new); } final Predicate predicate; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 292bafefbb6..2acb4138d23 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -1073,7 +1073,7 @@ public class SystemVmTemplateRegistration { } Hypervisor.HypervisorType hypervisorType = templateDetails.getHypervisorType(); updateSystemVMEntries(templateId, hypervisorType); - updateConfigurationParams(hypervisorType, templateDetails.getName(), zoneId); + updateConfigurationParams(hypervisorType, templateVO.getName(), zoneId); } protected void updateTemplateUrlChecksumAndGuestOsId(VMTemplateVO templateVO, diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java index 68100e16401..e0aa38a717b 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42020to42030.java @@ -57,8 +57,4 @@ public class Upgrade42020to42030 extends DbUpgradeAbstractImpl implements DbUpgr public InputStream[] getCleanupScripts() { return null; } - - @Override - public void updateSystemVmTemplates(Connection conn) { - } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java index df4743894c9..393f2039950 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42300.java @@ -17,7 +17,12 @@ package com.cloud.upgrade.dao; import java.io.InputStream; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import com.cloud.utils.crypt.DBEncryptionUtil; import com.cloud.utils.exception.CloudRuntimeException; public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgrade, DbUpgradeSystemVmTemplate { @@ -42,4 +47,46 @@ public class Upgrade42210to42300 extends DbUpgradeAbstractImpl implements DbUpgr return new InputStream[] {script}; } + + @Override + public void performDataMigration(Connection conn) { + unhideJsInterpretationEnabled(conn); + } + + protected void unhideJsInterpretationEnabled(Connection conn) { + String value = getJsInterpretationEnabled(conn); + if (value != null) { + updateJsInterpretationEnabledFields(conn, value); + } + } + + protected String getJsInterpretationEnabled(Connection conn) { + String query = "SELECT value FROM cloud.configuration WHERE name = 'js.interpretation.enabled' AND category = 'Hidden';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + ResultSet rs = pstmt.executeQuery(); + if (rs.next()) { + return rs.getString("value"); + } + logger.debug("Unable to retrieve value of hidden configuration 'js.interpretation.enabled'. The configuration may already be unhidden."); + return null; + } catch (SQLException e) { + throw new CloudRuntimeException("Error while retrieving value of hidden configuration 'js.interpretation.enabled'.", e); + } + } + + protected void updateJsInterpretationEnabledFields(Connection conn, String encryptedValue) { + String query = "UPDATE cloud.configuration SET value = ?, category = 'System', component = 'JsInterpreter', is_dynamic = 1 WHERE name = 'js.interpretation.enabled';"; + + try (PreparedStatement pstmt = conn.prepareStatement(query)) { + String decryptedValue = DBEncryptionUtil.decrypt(encryptedValue); + logger.info("Updating setting 'js.interpretation.enabled' to decrypted value [{}], category 'System', component 'JsInterpreter', and is_dynamic '1'.", decryptedValue); + pstmt.setString(1, decryptedValue); + pstmt.executeUpdate(); + } catch (SQLException e) { + throw new CloudRuntimeException("Error while unhiding configuration 'js.interpretation.enabled'.", e); + } catch (CloudRuntimeException e) { + logger.warn("Error while decrypting configuration 'js.interpretation.enabled'. The configuration may already be decrypted."); + } + } } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 23541c2431e..4fd3e729e0d 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -21,6 +21,7 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.Pair; @@ -192,4 +193,8 @@ public interface VMInstanceDao extends GenericDao, StateDao< int getVmCountByOfferingNotInDomain(Long serviceOfferingId, List domainIds); List listByIdsIncludingRemoved(List ids); + + List listDeleteProtectedVmsByAccountId(long accountId); + + List listDeleteProtectedVmsByDomainIds(Set domainIds); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index 589a63ea0d8..019d152ce5c 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -25,11 +25,13 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.cloudstack.api.ApiConstants; import org.apache.commons.collections.CollectionUtils; import org.springframework.stereotype.Component; @@ -106,6 +108,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem protected SearchBuilder IdsPowerStateSelectSearch; GenericSearchBuilder CountByOfferingId; GenericSearchBuilder CountUserVmNotInDomain; + SearchBuilder DeleteProtectedVmSearchByAccount; + SearchBuilder DeleteProtectedVmSearchByDomainIds; @Inject ResourceTagDao tagsDao; @@ -354,7 +358,8 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem IdsPowerStateSelectSearch.entity().getPowerHostId(), IdsPowerStateSelectSearch.entity().getPowerState(), IdsPowerStateSelectSearch.entity().getPowerStateUpdateCount(), - IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime()); + IdsPowerStateSelectSearch.entity().getPowerStateUpdateTime(), + IdsPowerStateSelectSearch.entity().getState()); IdsPowerStateSelectSearch.done(); CountByOfferingId = createSearchBuilder(Integer.class); @@ -368,6 +373,19 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem CountUserVmNotInDomain.and("domainIdsNotIn", CountUserVmNotInDomain.entity().getDomainId(), Op.NIN); CountUserVmNotInDomain.done(); + DeleteProtectedVmSearchByAccount = createSearchBuilder(); + DeleteProtectedVmSearchByAccount.selectFields(DeleteProtectedVmSearchByAccount.entity().getUuid()); + DeleteProtectedVmSearchByAccount.and(ApiConstants.ACCOUNT_ID, DeleteProtectedVmSearchByAccount.entity().getAccountId(), Op.EQ); + DeleteProtectedVmSearchByAccount.and(ApiConstants.DELETE_PROTECTION, DeleteProtectedVmSearchByAccount.entity().isDeleteProtection(), Op.EQ); + DeleteProtectedVmSearchByAccount.and(ApiConstants.REMOVED, DeleteProtectedVmSearchByAccount.entity().getRemoved(), Op.NULL); + DeleteProtectedVmSearchByAccount.done(); + + DeleteProtectedVmSearchByDomainIds = createSearchBuilder(); + DeleteProtectedVmSearchByDomainIds.selectFields(DeleteProtectedVmSearchByDomainIds.entity().getUuid()); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.DOMAIN_IDS, DeleteProtectedVmSearchByDomainIds.entity().getDomainId(), Op.IN); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.DELETE_PROTECTION, DeleteProtectedVmSearchByDomainIds.entity().isDeleteProtection(), Op.EQ); + DeleteProtectedVmSearchByDomainIds.and(ApiConstants.REMOVED, DeleteProtectedVmSearchByDomainIds.entity().getRemoved(), Op.NULL); + DeleteProtectedVmSearchByDomainIds.done(); } @Override @@ -1088,10 +1106,14 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private boolean isPowerStateInSyncWithInstanceState(final VirtualMachine.PowerState powerState, final long powerHostId, final VMInstanceVO instance) { State instanceState = instance.getState(); + if (instanceState == null) { + logger.warn("VM {} has null instance state during power state sync check, treating as out of sync", instance); + return false; + } if ((powerState == VirtualMachine.PowerState.PowerOff && instanceState == State.Running) || (powerState == VirtualMachine.PowerState.PowerOn && instanceState == State.Stopped)) { HostVO instanceHost = hostDao.findById(instance.getHostId()); - HostVO powerHost = powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); + HostVO powerHost = instance.getHostId() != null && powerHostId == instance.getHostId() ? instanceHost : hostDao.findById(powerHostId); logger.debug("VM: {} on host: {} and power host : {} is in {} state, but power state is {}", instance, instanceHost, powerHost, instanceState, powerState); return false; @@ -1296,4 +1318,22 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem sc.setParameters("ids", ids.toArray()); return listIncludingRemovedBy(sc); } + + @Override + public List listDeleteProtectedVmsByAccountId(long accountId) { + SearchCriteria sc = DeleteProtectedVmSearchByAccount.create(); + sc.setParameters(ApiConstants.ACCOUNT_ID, accountId); + sc.setParameters(ApiConstants.DELETE_PROTECTION, true); + Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); + return listBy(sc, filter); + } + + @Override + public List listDeleteProtectedVmsByDomainIds(Set domainIds) { + SearchCriteria sc = DeleteProtectedVmSearchByDomainIds.create(); + sc.setParameters(ApiConstants.DOMAIN_IDS, domainIds.toArray()); + sc.setParameters(ApiConstants.DELETE_PROTECTION, true); + Filter filter = new Filter(VMInstanceVO.class, null, false, 0L, 10L); + return listBy(sc, filter); + } } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java index ee1783a9c89..87b7dab1ff7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDao.java @@ -21,20 +21,14 @@ import java.util.Date; import java.util.List; import com.cloud.utils.DateUtil; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.backup.BackupSchedule; import org.apache.cloudstack.backup.BackupScheduleVO; import com.cloud.utils.db.GenericDao; public interface BackupScheduleDao extends GenericDao { - BackupScheduleVO findByVM(Long vmId); - List listByVM(Long vmId); BackupScheduleVO findByVMAndIntervalType(Long vmId, DateUtil.IntervalType intervalType); List getSchedulesToExecute(Date currentTimestamp); - - BackupScheduleResponse newBackupScheduleResponse(BackupSchedule schedule); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java index d9cf7b63680..972af73391a 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/dao/BackupScheduleDaoImpl.java @@ -17,28 +17,23 @@ package org.apache.cloudstack.backup.dao; +import java.sql.PreparedStatement; +import java.sql.SQLException; import java.util.Date; import java.util.List; import javax.annotation.PostConstruct; -import javax.inject.Inject; import com.cloud.utils.DateUtil; -import org.apache.cloudstack.api.response.BackupScheduleResponse; -import org.apache.cloudstack.backup.BackupSchedule; +import com.cloud.utils.db.DB; +import com.cloud.utils.db.TransactionLegacy; import org.apache.cloudstack.backup.BackupScheduleVO; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.vm.VMInstanceVO; -import com.cloud.vm.dao.VMInstanceDao; public class BackupScheduleDaoImpl extends GenericDaoBase implements BackupScheduleDao { - - @Inject - VMInstanceDao vmInstanceDao; - private SearchBuilder backupScheduleSearch; private SearchBuilder executableSchedulesSearch; @@ -59,13 +54,6 @@ public class BackupScheduleDaoImpl extends GenericDaoBase sc = backupScheduleSearch.create(); - sc.setParameters("vm_id", vmId); - return findOneBy(sc); - } - @Override public List listByVM(Long vmId) { SearchCriteria sc = backupScheduleSearch.create(); @@ -88,21 +76,19 @@ public class BackupScheduleDaoImpl extends GenericDaoBase listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, ObjectInDataStoreStateMachine.State... state); + List listReadyByVolumeIdAndCheckpointPathNotNull(long volumeId); SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index bbb2b4f3a88..8b7a2b78de7 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -68,6 +68,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase searchFilteringStoreIdEqStateEqStoreRoleEqIdEqUpdateCountEqSnapshotIdEqVolumeIdEq; private SearchBuilder stateSearch; private SearchBuilder idStateNinSearch; + private SearchBuilder idEqRoleEqStateInSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; private SearchBuilder dataStoreAndInstallPathSearch; @@ -151,6 +152,11 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase listBySnapshotIdAndDataStoreRoleAndStateIn(long snapshotId, DataStoreRole role, State... state) { + SearchCriteria sc = idEqRoleEqStateInSearch.create(); + sc.setParameters(SNAPSHOT_ID, snapshotId); + sc.setParameters(STORE_ROLE, role); + sc.setParameters(STATE, (Object[])state); + return listBy(sc); + } + @Override public SnapshotDataStoreVO findOneBySnapshotId(long snapshotId, long zoneId) { try (TransactionLegacy transactionLegacy = TransactionLegacy.currentTxn()) { diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml index 1846c3c62a0..2c6869bd81e 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml @@ -73,5 +73,7 @@ + + diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index edc14d9fa0c..ad3722577c2 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -236,13 +236,11 @@ - - diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql index 3dd6c18f57c..4405250f271 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42000to42010.sql @@ -31,7 +31,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.account', 'api_key_access', 'boolean CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.account', 'api_key_access', 'boolean DEFAULT NULL COMMENT "is api key access allowed for the account" '); -- Create a new group for Usage Server related configurations -INSERT INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); +INSERT IGNORE INTO `cloud`.`configuration_group` (`name`, `description`, `precedence`) VALUES ('Usage Server', 'Usage Server related configuration', 9); UPDATE `cloud`.`configuration_subgroup` set `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server'), `precedence` = 1 WHERE `name`='Usage'; UPDATE `cloud`.`configuration` SET `group_id` = (SELECT `id` FROM `cloud`.`configuration_group` WHERE `name` = 'Usage Server') where `subgroup_id` = (SELECT `id` FROM `cloud`.`configuration_subgroup` WHERE `name` = 'Usage'); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql index 4c65f37e0fe..000b54b7207 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42010to42100.sql @@ -687,22 +687,23 @@ CREATE TABLE IF NOT EXISTS `cloud`.`backup_details` ( UPDATE `cloud`.`backups` b INNER JOIN `cloud`.`vm_instance` vm ON b.vm_id = vm.id SET b.backed_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id @@ -711,22 +712,23 @@ SET b.backed_volumes = ( -- Add diskOfferingId, deviceId, minIops and maxIops to backup_volumes in vm_instance table UPDATE `cloud`.`vm_instance` vm SET vm.backup_volumes = ( - SELECT CONCAT("[", - GROUP_CONCAT( - CONCAT( - "{\"uuid\":\"", v.uuid, "\",", - "\"type\":\"", v.volume_type, "\",", - "\"size\":", v.`size`, ",", - "\"path\":\"", IFNULL(v.path, 'null'), "\",", - "\"deviceId\":", IFNULL(v.device_id, 'null'), ",", - "\"diskOfferingId\":\"", doff.uuid, "\",", - "\"minIops\":", IFNULL(v.min_iops, 'null'), ",", - "\"maxIops\":", IFNULL(v.max_iops, 'null'), - "}" - ) - SEPARATOR "," + SELECT COALESCE( + CAST( + JSON_ARRAYAGG( + JSON_OBJECT( + 'uuid', v.uuid, + 'type', v.volume_type, + 'size', v.size, + 'path', v.path, + 'deviceId', v.device_id, + 'diskOfferingId', doff.uuid, + 'minIops', v.min_iops, + 'maxIops', v.max_iops + ) + ) AS CHAR ), - "]") + '[]' + ) FROM `cloud`.`volumes` v LEFT JOIN `cloud`.`disk_offering` doff ON v.disk_offering_id = doff.id WHERE v.instance_id = vm.id diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql index 505c8ef5715..2f104568c14 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210-cleanup.sql @@ -19,4 +19,8 @@ -- Schema upgrade cleanup from 4.22.0.0 to 4.22.1.0 --; -DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; +-- Entries remaining on `cloud`.`resource_reservation` during the upgrade process are stale, so delete them. +-- This script was added to normalize volume/primary storage reservations that got stuck due to a bug on VM deployment, +-- but it is more interesting to introduce a smarter logic to clean these stale reservations in the future without the need +-- for upgrades (for instance, by having a heartbeat_time column for the reservations and automatically cleaning old entries). +DELETE FROM `cloud`.`resource_reservation`; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql index 2326a855f32..baa20e9f0ad 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42200to42210.sql @@ -34,4 +34,28 @@ UPDATE `cloud`.`alert` SET type = 34 WHERE name = 'ALERT.VR.PRIVATE.IFACE.MTU'; -- Update configuration 'kvm.ssh.to.agent' description and is_dynamic fields UPDATE `cloud`.`configuration` SET description = 'True if the management server will restart the agent service via SSH into the KVM hosts after or during maintenance operations', is_dynamic = 1 WHERE name = 'kvm.ssh.to.agent'; +-- Sanitize legacy network-level addressing fields for Public networks +UPDATE `cloud`.`networks` +SET `broadcast_uri` = NULL, + `gateway` = NULL, + `cidr` = NULL, + `ip6_gateway` = NULL, + `ip6_cidr` = NULL +WHERE `traffic_type` = 'Public'; + UPDATE `cloud`.`vm_template` SET guest_os_id = 99 WHERE name = 'kvm-default-vm-import-dummy-template'; + +-- Update existing vm_template records with NULL type to "USER" +UPDATE `cloud`.`vm_template` SET `type` = 'USER' WHERE `type` IS NULL; + +-- remove unused config item +DELETE FROM `cloud`.`configuration` WHERE name = 'consoleproxy.cmd.port'; + +-- Drops the unused "backup_interval_type" column of the "cloud.backups" table +ALTER TABLE `cloud`.`backups` DROP COLUMN `backup_interval_type`; + +-- Update `user.password.reset.mail.template` configuration value to match new logic +UPDATE `cloud`.`configuration` +SET value = CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team') +WHERE name = 'user.password.reset.mail.template' + AND value IN (CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', 'http://{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team'), CONCAT_WS('\n', 'Hello {{username}}!', 'You have requested to reset your password. Please click the following link to reset your password:', '{{{domainUrl}}}{{{resetLink}}}', 'If you did not request a password reset, please ignore this email.', '', 'Regards,', 'The CloudStack Team')); diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql index 4cb9eb7cb2c..dfa5fa8f3a1 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42300.sql @@ -117,3 +117,35 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc_offerings','conserve_mode', 'tin --- Disable/enable NICs CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.nics','enabled', 'TINYINT(1) NOT NULL DEFAULT 1 COMMENT ''Indicates whether the NIC is enabled or not'' '); + +--- Quota tariff/usage mapping +CREATE TABLE IF NOT EXISTS `cloud_usage`.`quota_tariff_usage` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `tariff_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the tariff of the Quota usage detail calculated, foreign key to quota_tariff table', + `quota_usage_id` bigint(20) unsigned NOT NULL COMMENT 'ID of the aggregation of Quota usage details, foreign key to quota_usage table', + `quota_used` decimal(20,8) NOT NULL COMMENT 'Amount of quota used', + PRIMARY KEY (`id`), + CONSTRAINT `fk_quota_tariff_usage__tariff_id` FOREIGN KEY (`tariff_id`) REFERENCES `cloud_usage`.`quota_tariff` (`id`), + CONSTRAINT `fk_quota_tariff_usage__quota_usage_id` FOREIGN KEY (`quota_usage_id`) REFERENCES `cloud_usage`.`quota_usage` (`id`)); + +-- Add the 'keep_mac_address_on_public_nic' column to the 'cloud.networks' and 'cloud.vpc' tables +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.networks', 'keep_mac_address_on_public_nic', 'TINYINT(1) NOT NULL DEFAULT 1'); +CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.vpc', 'keep_mac_address_on_public_nic', 'TINYINT(1) NOT NULL DEFAULT 1'); + +-- Creates the 'kvm.memory.dynamic.scaling.capacity' and, for already active ACS environments, +-- initializes it with the value of the setting 'vm.serviceoffering.ram.size.max' +INSERT INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `updated`, `scope`, `is_dynamic`, `group_id`, `subgroup_id`, `display_text`, `description`) +SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.memory.dynamic.scaling.capacity', `cfg`.`value`, 0, NULL, 4, 1, 6, 27, + 'KVM memory dynamic scaling capacity', 'Defines the maximum memory capacity in MiB for which VMs can be dynamically scaled to with KVM. The ''kvm.memory.dynamic.scaling.capacity'' setting''s value will be used to define the value of the '''' element of domain XMLs. If it is set to a value less than or equal to ''0'', then the host''s memory capacity will be considered.' +FROM `cloud`.`configuration` `cfg` +WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.memory.dynamic.scaling.capacity') + AND `cfg`.`name` = 'vm.serviceoffering.ram.size.max'; + +-- Creates the 'kvm.cpu.dynamic.scaling.capacity' and, for already active ACS environments, +-- initializes it with the value of the setting 'vm.serviceoffering.cpu.cores.max' +INSERT INTO `cloud`.`configuration` (`category`, `instance`, `component`, `name`, `value`, `default_value`, `updated`, `scope`, `is_dynamic`, `group_id`, `subgroup_id`, `display_text`, `description`) +SELECT 'Advanced', 'DEFAULT', 'CapacityManager', 'kvm.cpu.dynamic.scaling.capacity', `cfg`.`value`, 0, NULL, 4, 1, 6, 27, + 'KVM CPU dynamic scaling capacity', 'Defines the maximum vCPU capacity for which VMs can be dynamically scaled to with KVM. The ''kvm.cpu.dynamic.scaling.capacity'' setting''s value will be used to define the value of the '''' element of domain XMLs. If it is set to a value less than or equal to ''0'', then the host''s CPU cores capacity will be considered.' +FROM `cloud`.`configuration` `cfg` +WHERE NOT EXISTS (SELECT 1 FROM `cloud`.`configuration` WHERE `name` = 'kvm.cpu.dynamic.scaling.capacity') + AND `cfg`.`name` = 'vm.serviceoffering.cpu.cores.max'; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql new file mode 100644 index 00000000000..11193c465fd --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_netstats_view.sql @@ -0,0 +1,31 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- cloud.account_netstats_view source + + +DROP VIEW IF EXISTS `cloud`.`account_netstats_view`; + +CREATE VIEW `cloud`.`account_netstats_view` AS +select + `user_statistics`.`account_id` AS `account_id`, + (sum(`user_statistics`.`net_bytes_received`) + sum(`user_statistics`.`current_bytes_received`)) AS `bytesReceived`, + (sum(`user_statistics`.`net_bytes_sent`) + sum(`user_statistics`.`current_bytes_sent`)) AS `bytesSent` +from + `user_statistics` +group by + `user_statistics`.`account_id`; diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql index 327c6c627e2..edc164c40cb 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.account_view.sql @@ -39,8 +39,8 @@ select `data_center`.`id` AS `data_center_id`, `data_center`.`uuid` AS `data_center_uuid`, `data_center`.`name` AS `data_center_name`, - `account_netstats`.`bytesReceived` AS `bytesReceived`, - `account_netstats`.`bytesSent` AS `bytesSent`, + `account_netstats_view`.`bytesReceived` AS `bytesReceived`, + `account_netstats_view`.`bytesSent` AS `bytesSent`, `vmlimit`.`max` AS `vmLimit`, `vmcount`.`count` AS `vmTotal`, `runningvm`.`vmcount` AS `runningVms`, @@ -89,15 +89,8 @@ from `cloud`.`domain` ON account.domain_id = domain.id left join `cloud`.`data_center` ON account.default_zone_id = data_center.id - left join lateral ( - select - coalesce(sum(`user_statistics`.`net_bytes_received` + `user_statistics`.`current_bytes_received`), 0) AS `bytesReceived`, - coalesce(sum(`user_statistics`.`net_bytes_sent` + `user_statistics`.`current_bytes_sent`), 0) AS `bytesSent` - from - `cloud`.`user_statistics` - where - `user_statistics`.`account_id` = `account`.`id` - ) AS `account_netstats` ON TRUE + left join + `cloud`.`account_netstats_view` ON account.id = account_netstats_view.account_id left join `cloud`.`resource_limit` vmlimit ON account.id = vmlimit.account_id and vmlimit.type = 'user_vm' and vmlimit.tag IS NULL diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql index 6f31fc17bce..c169cbce217 100644 --- a/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud.user_vm_view.sql @@ -79,6 +79,7 @@ SELECT `vm_template`.`format` AS `template_format`, `vm_template`.`display_text` AS `template_display_text`, `vm_template`.`enable_password` AS `password_enabled`, + `vm_template`.`extension_id` AS `template_extension_id`, `iso`.`id` AS `iso_id`, `iso`.`uuid` AS `iso_uuid`, `iso`.`name` AS `iso_name`, diff --git a/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql new file mode 100644 index 00000000000..7ac001384e8 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/views/cloud_usage.quota_usage_view.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- VIEW `cloud_usage`.`quota_usage_view`; + +DROP VIEW IF EXISTS `cloud_usage`.`quota_usage_view`; +CREATE VIEW `cloud_usage`.`quota_usage_view` AS +SELECT qu.id, + qu.usage_item_id, + qu.zone_id, + qu.account_id, + qu.domain_id, + qu.usage_type, + qu.quota_used, + qu.start_date, + qu.end_date, + cu.usage_id AS resource_id, + cu.network_id as network_id, + cu.offering_id as offering_id +FROM `cloud_usage`.`quota_usage` qu +INNER JOIN `cloud_usage`.`cloud_usage` cu ON (cu.id = qu.usage_item_id); diff --git a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java index 27995eb179a..ab64e4698f0 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/DatabaseUpgradeCheckerTest.java @@ -44,6 +44,7 @@ import com.cloud.upgrade.dao.Upgrade41120to41130; import com.cloud.upgrade.dao.Upgrade41120to41200; import com.cloud.upgrade.dao.Upgrade41510to41520; import com.cloud.upgrade.dao.Upgrade41610to41700; +import com.cloud.upgrade.dao.Upgrade42010to42100; import com.cloud.upgrade.dao.Upgrade452to453; import com.cloud.upgrade.dao.Upgrade453to460; import com.cloud.upgrade.dao.Upgrade460to461; @@ -380,4 +381,23 @@ public class DatabaseUpgradeCheckerTest { assertFalse("DatabaseUpgradeChecker should not be a standalone component", checker.isStandalone()); } + @Test + public void testCalculateUpgradePath42010to42100() { + + final CloudStackVersion dbVersion = CloudStackVersion.parse("4.20.1.0"); + assertNotNull(dbVersion); + + final CloudStackVersion currentVersion = CloudStackVersion.parse("4.21.0.0"); + assertNotNull(currentVersion); + + final DatabaseUpgradeChecker checker = new DatabaseUpgradeChecker(); + final DbUpgrade[] upgrades = checker.calculateUpgradePath(dbVersion, currentVersion); + + assertNotNull(upgrades); + assertEquals(1, upgrades.length); + assertTrue(upgrades[0] instanceof Upgrade42010to42100); + + assertArrayEquals(new String[]{"4.20.1.0", "4.21.0.0"}, upgrades[0].getUpgradableVersionRange()); + assertEquals(currentVersion.toString(), upgrades[0].getUpgradedVersion()); + } } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index 8028e78c907..51db952eb61 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -570,18 +570,19 @@ public class SystemVmTemplateRegistrationTest { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + String templateName = "templateName"; + when(templateVO.getName()).thenReturn(templateName); GuestOSVO guestOS = Mockito.mock(GuestOSVO.class); when(templateDetails.getGuestOs()).thenReturn("Debian"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(templateDetails.getName()).thenReturn("templateName"); when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); when(guestOSDao.findOneByDisplayName("Debian")).thenReturn(guestOS); when(guestOS.getId()).thenReturn(10L); when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); @@ -590,7 +591,7 @@ public class SystemVmTemplateRegistrationTest { verify(vmTemplateDao).update(templateVO.getId(), templateVO); verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); } @Test @@ -620,16 +621,17 @@ public class SystemVmTemplateRegistrationTest { SystemVmTemplateRegistration.MetadataTemplateDetails templateDetails = Mockito.mock(SystemVmTemplateRegistration.MetadataTemplateDetails.class); VMTemplateVO templateVO = Mockito.mock(VMTemplateVO.class); + String templateName = "templateName"; + when(templateVO.getName()).thenReturn(templateName); when(templateDetails.getGuestOs()).thenReturn("NonExistentOS"); when(templateDetails.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); - when(templateDetails.getName()).thenReturn("templateName"); when(vmTemplateDao.findById(templateId)).thenReturn(templateVO); when(guestOSDao.findOneByDisplayName("NonExistentOS")).thenReturn(null); when(vmTemplateDao.update(templateVO.getId(), templateVO)).thenReturn(true); doNothing().when(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); doNothing().when(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); systemVmTemplateRegistration.updateRegisteredTemplateDetails(templateId, templateDetails, zoneId); @@ -637,7 +639,7 @@ public class SystemVmTemplateRegistrationTest { verify(vmTemplateDao).update(templateVO.getId(), templateVO); verify(systemVmTemplateRegistration).updateSystemVMEntries(templateId, Hypervisor.HypervisorType.KVM); verify(systemVmTemplateRegistration).updateConfigurationParams(Hypervisor.HypervisorType.KVM, - "templateName", zoneId); + templateName, zoneId); } @Test diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java index 7174336113b..88f479c0904 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategy.java @@ -120,7 +120,7 @@ public class DefaultSnapshotStrategy extends SnapshotStrategyBase { private final List snapshotStatesAbleToDeleteSnapshot = Arrays.asList(Snapshot.State.Destroying, Snapshot.State.Destroyed, Snapshot.State.Error, Snapshot.State.Hidden); public SnapshotDataStoreVO getSnapshotImageStoreRef(long snapshotId, long zoneId) { - List snaps = snapshotStoreDao.listReadyBySnapshot(snapshotId, DataStoreRole.Image); + List snaps = snapshotStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(snapshotId, DataStoreRole.Image, State.Ready, State.Hidden); for (SnapshotDataStoreVO ref : snaps) { if (zoneId == dataStoreMgr.getStoreZoneId(ref.getDataStoreId(), ref.getRole())) { return ref; diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java index 7199fce1d34..aced750bd32 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java @@ -40,6 +40,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.util.ScaleIOUtil; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.cloudstack.storage.datastore.api.Volume; import com.cloud.agent.api.VMSnapshotTO; import com.cloud.alert.AlertManager; @@ -200,11 +201,35 @@ public class ScaleIOVMSnapshotStrategy extends ManagerBase implements VMSnapshot if (volumeIds != null && !volumeIds.isEmpty()) { List vmSnapshotDetails = new ArrayList(); vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "SnapshotGroupId", snapshotGroupId, false)); + Map snapshotNameToSrcPathMap = new HashMap<>(); + for (Map.Entry entry : srcVolumeDestSnapshotMap.entrySet()) { + snapshotNameToSrcPathMap.put(entry.getValue(), entry.getKey()); + } - for (int index = 0; index < volumeIds.size(); index++) { - String volumeSnapshotName = srcVolumeDestSnapshotMap.get(ScaleIOUtil.getVolumePath(volumeTOs.get(index).getPath())); - String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(volumeIds.get(index), volumeSnapshotName); - vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), "Vol_" + volumeTOs.get(index).getId() + "_Snapshot", pathWithScaleIOVolumeName, false)); + for (String snapshotVolumeId : volumeIds) { + // Use getVolume() to fetch snapshot volume details and get its name + Volume snapshotVolume = client.getVolume(snapshotVolumeId); + if (snapshotVolume == null) { + throw new CloudRuntimeException("Cannot find snapshot volume with id: " + snapshotVolumeId); + } + String snapshotName = snapshotVolume.getName(); + + // Match back to source volume path + String srcVolumePath = snapshotNameToSrcPathMap.get(snapshotName); + if (srcVolumePath == null) { + throw new CloudRuntimeException("Cannot match snapshot " + snapshotName + " to a source volume"); + } + + // Find the matching VolumeObjectTO by path + VolumeObjectTO matchedVolume = volumeTOs.stream() + .filter(v -> ScaleIOUtil.getVolumePath(v.getPath()).equals(srcVolumePath)) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException("Cannot find source volume for path: " + srcVolumePath)); + + String pathWithScaleIOVolumeName = ScaleIOUtil.updatedPathWithVolumeName(snapshotVolumeId, snapshotName); + vmSnapshotDetails.add(new VMSnapshotDetailsVO(vmSnapshot.getId(), + "Vol_" + matchedVolume.getId() + "_Snapshot", + pathWithScaleIOVolumeName, false)); } vmSnapshotDetailsDao.saveDetails(vmSnapshotDetails); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java index e3f28a7012c..31b13fc279e 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/StorageVMSnapshotStrategy.java @@ -111,7 +111,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { FreezeThawVMAnswer freezeAnswer = null; FreezeThawVMCommand thawCmd = null; FreezeThawVMAnswer thawAnswer = null; - List forRollback = new ArrayList<>(); + List snapshotsForRollback = new ArrayList<>(); long startFreeze = 0; try { vmSnapshotHelper.vmSnapshotStateTransitTo(vmSnapshotVO, VMSnapshot.Event.CreateRequested); @@ -165,7 +165,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { logger.info("The virtual machine is frozen"); for (VolumeInfo vol : vinfos) { long startSnapshtot = System.nanoTime(); - SnapshotInfo snapInfo = createDiskSnapshot(vmSnapshot, forRollback, vol); + SnapshotInfo snapInfo = createDiskSnapshot(vmSnapshot, snapshotsForRollback, vol); if (snapInfo == null) { thawAnswer = (FreezeThawVMAnswer) agentMgr.send(hostId, thawCmd); @@ -222,7 +222,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { } } if (!result) { - for (SnapshotInfo snapshotInfo : forRollback) { + for (SnapshotInfo snapshotInfo : snapshotsForRollback) { rollbackDiskSnapshot(snapshotInfo); } try { @@ -388,10 +388,16 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { //Rollback if one of disks snapshot fails protected void rollbackDiskSnapshot(SnapshotInfo snapshotInfo) { + if (snapshotInfo == null) { + return; + } Long snapshotID = snapshotInfo.getId(); SnapshotVO snapshot = snapshotDao.findById(snapshotID); + if (snapshot == null) { + return; + } deleteSnapshotByStrategy(snapshot); - logger.debug("Rollback is executed: deleting snapshot with id:" + snapshotID); + logger.debug("Rollback is executed: deleting snapshot with id: {}", snapshotID); } protected void deleteSnapshotByStrategy(SnapshotVO snapshot) { @@ -434,7 +440,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { } } - protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List forRollback, VolumeInfo vol) { + protected SnapshotInfo createDiskSnapshot(VMSnapshot vmSnapshot, List snapshotsForRollback, VolumeInfo vol) { String snapshotName = vmSnapshot.getId() + "_" + vol.getUuid(); SnapshotVO snapshot = new SnapshotVO(vol.getDataCenterId(), vol.getAccountId(), vol.getDomainId(), vol.getId(), vol.getDiskOfferingId(), snapshotName, (short) Snapshot.Type.GROUP.ordinal(), Snapshot.Type.GROUP.name(), vol.getSize(), vol.getMinIops(), vol.getMaxIops(), Hypervisor.HypervisorType.KVM, null); @@ -448,6 +454,7 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { vol.addPayload(setPayload(vol, snapshot, quiescevm)); SnapshotInfo snapshotInfo = snapshotDataFactory.getSnapshot(snapshot.getId(), vol.getDataStore()); snapshotInfo.addPayload(vol.getpayload()); + snapshotsForRollback.add(snapshotInfo); SnapshotStrategy snapshotStrategy = storageStrategyFactory.getSnapshotStrategy(snapshotInfo, SnapshotOperation.TAKE); if (snapshotStrategy == null) { throw new CloudRuntimeException("Could not find strategy for snapshot uuid:" + snapshotInfo.getUuid()); @@ -455,8 +462,6 @@ public class StorageVMSnapshotStrategy extends DefaultVMSnapshotStrategy { snapshotInfo = snapshotStrategy.takeSnapshot(snapshotInfo); if (snapshotInfo == null) { throw new CloudRuntimeException("Failed to create snapshot"); - } else { - forRollback.add(snapshotInfo); } vmSnapshotDetailsDao.persist(new VMSnapshotDetailsVO(vmSnapshot.getId(), STORAGE_SNAPSHOT, String.valueOf(snapshot.getId()), true)); snapshotInfo.markBackedUp(); diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java index 53f98c18f1b..41bfaa6f0c7 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/snapshot/DefaultSnapshotStrategyTest.java @@ -257,11 +257,6 @@ public class DefaultSnapshotStrategyTest { @Test public void testGetSnapshotImageStoreRefNull() { - SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); - Mockito.when(ref1.getDataStoreId()).thenReturn(1L); - Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); - Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); - Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(2L); Assert.assertNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); } @@ -270,7 +265,7 @@ public class DefaultSnapshotStrategyTest { SnapshotDataStoreVO ref1 = Mockito.mock(SnapshotDataStoreVO.class); Mockito.when(ref1.getDataStoreId()).thenReturn(1L); Mockito.when(ref1.getRole()).thenReturn(DataStoreRole.Image); - Mockito.when(snapshotDataStoreDao.listReadyBySnapshot(Mockito.anyLong(), Mockito.any(DataStoreRole.class))).thenReturn(List.of(ref1)); + Mockito.when(snapshotDataStoreDao.listBySnapshotIdAndDataStoreRoleAndStateIn(Mockito.anyLong(), Mockito.any(DataStoreRole.class), Mockito.any(), Mockito.any())).thenReturn(List.of(ref1)); Mockito.when(dataStoreManager.getStoreZoneId(1L, DataStoreRole.Image)).thenReturn(1L); Assert.assertNotNull(defaultSnapshotStrategySpy.getSnapshotImageStoreRef(1L, 1L)); } diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java index 050c0246aba..7d5d3c786e8 100644 --- a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/VMSnapshotStrategyKVMTest.java @@ -155,7 +155,7 @@ public class VMSnapshotStrategyKVMTest extends TestCase{ @Test public void testCreateDiskSnapshotBasedOnStrategy() throws Exception { VMSnapshotVO vmSnapshot = Mockito.mock(VMSnapshotVO.class); - List forRollback = new ArrayList<>(); + List snapshotsForRollback = new ArrayList<>(); VolumeInfo vol = Mockito.mock(VolumeInfo.class); SnapshotInfo snapshotInfo = Mockito.mock(SnapshotInfo.class); SnapshotStrategy strategy = Mockito.mock(SnapshotStrategy.class); @@ -179,7 +179,7 @@ public class VMSnapshotStrategyKVMTest extends TestCase{ VMSnapshotDetailsVO vmDetails = new VMSnapshotDetailsVO(vmSnapshot.getId(), volUuid, String.valueOf(snapshot.getId()), false); when(vmSnapshotDetailsDao.persist(any())).thenReturn(vmDetails); - info = vmStrategy.createDiskSnapshot(vmSnapshot, forRollback, vol); + info = vmStrategy.createDiskSnapshot(vmSnapshot, snapshotsForRollback, vol); assertNotNull(info); } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java index cd9ab3adb21..4057f7a051b 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/allocator/AbstractStoragePoolAllocator.java @@ -145,10 +145,10 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement storageType = "shared"; } - logger.debug(String.format( - "Filtering storage pools by capacity type [%s] as the first storage pool of the list, with name [%s] and ID [%s], is a [%s] storage.", + logger.info( + "Filtering storage pools by capacity type [{}] as the first storage pool of the list, with name [{}] and ID [{}], is a [{}] storage.", capacityType, storagePool.getName(), storagePool.getUuid(), storageType - )); + ); Pair, Map> result = capacityDao.orderHostsByFreeCapacity(zoneId, clusterId, capacityType); List poolIdsByCapacity = result.first(); @@ -185,7 +185,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement Long clusterId = plan.getClusterId(); List poolIdsByVolCount = volumeDao.listPoolIdsByVolumeCount(dcId, podId, clusterId, account.getAccountId()); - logger.debug(String.format("List of pools in ascending order of number of volumes for account [%s] is [%s].", account, poolIdsByVolCount)); + logger.debug("List of pools in ascending order of number of volumes for account [{}] is [{}].", account, poolIdsByVolCount); // now filter the given list of Pools by this ordered list Map poolMap = new HashMap<>(); @@ -206,16 +206,11 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement @Override public List reorderPools(List pools, VirtualMachineProfile vmProfile, DeploymentPlan plan, DiskProfile dskCh) { - if (logger.isTraceEnabled()) { - logger.trace("reordering pools"); - } if (pools == null) { - logger.trace("There are no pools to reorder; returning null."); + logger.info("There are no pools to reorder."); return null; } - if (logger.isTraceEnabled()) { - logger.trace(String.format("reordering %d pools", pools.size())); - } + logger.info("Reordering [{}] pools", pools.size()); Account account = null; if (vmProfile.getVirtualMachine() != null) { account = vmProfile.getOwner(); @@ -224,9 +219,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement pools = reorderStoragePoolsBasedOnAlgorithm(pools, plan, account); if (vmProfile.getVirtualMachine() == null) { - if (logger.isTraceEnabled()) { - logger.trace("The VM is null, skipping pools reordering by disk provisioning type."); - } + logger.info("The VM is null, skipping pool reordering by disk provisioning type."); return pools; } @@ -240,14 +233,10 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement List reorderStoragePoolsBasedOnAlgorithm(List pools, DeploymentPlan plan, Account account) { String volumeAllocationAlgorithm = VolumeOrchestrationService.VolumeAllocationAlgorithm.value(); - logger.debug("Using volume allocation algorithm {} to reorder pools.", volumeAllocationAlgorithm); + logger.info("Using volume allocation algorithm {} to reorder pools.", volumeAllocationAlgorithm); if (volumeAllocationAlgorithm.equals("random") || (account == null)) { reorderRandomPools(pools); } else if (StringUtils.equalsAny(volumeAllocationAlgorithm, "userdispersing", "firstfitleastconsumed")) { - if (logger.isTraceEnabled()) { - logger.trace("Using reordering algorithm {}", volumeAllocationAlgorithm); - } - if (volumeAllocationAlgorithm.equals("userdispersing")) { pools = reorderPoolsByNumberOfVolumes(plan, pools, account); } else { @@ -259,16 +248,15 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement void reorderRandomPools(List pools) { StorageUtil.traceLogStoragePools(pools, logger, "pools to choose from: "); - if (logger.isTraceEnabled()) { - logger.trace("Shuffle this so that we don't check the pools in the same order. Algorithm == 'random' (or no account?)"); - } - StorageUtil.traceLogStoragePools(pools, logger, "pools to shuffle: "); + logger.trace("Shuffle this so that we don't check the pools in the same order. Algorithm == 'random' (or no account?)"); + logger.debug("Pools to shuffle: [{}]", pools); Collections.shuffle(pools, secureRandom); - StorageUtil.traceLogStoragePools(pools, logger, "shuffled list of pools to choose from: "); + logger.debug("Shuffled list of pools to choose from: [{}]", pools); } private List reorderPoolsByDiskProvisioningType(List pools, DiskProfile diskProfile) { if (diskProfile != null && diskProfile.getProvisioningType() != null && !diskProfile.getProvisioningType().equals(Storage.ProvisioningType.THIN)) { + logger.info("Reordering [{}] pools by disk provisioning type [{}].", pools.size(), diskProfile.getProvisioningType()); List reorderedPools = new ArrayList<>(); int preferredIndex = 0; for (StoragePool pool : pools) { @@ -282,22 +270,28 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement reorderedPools.add(preferredIndex++, pool); } } + logger.debug("Reordered list of pools by disk provisioning type [{}]: [{}]", diskProfile.getProvisioningType(), reorderedPools); return reorderedPools; } else { + if (diskProfile == null) { + logger.info("Reordering pools by disk provisioning type wasn't necessary, since no disk profile was found."); + } else { + logger.debug("Reordering pools by disk provisioning type wasn't necessary, since the provisioning type is [{}].", diskProfile.getProvisioningType()); + } return pools; } } protected boolean filter(ExcludeList avoid, StoragePool pool, DiskProfile dskCh, DeploymentPlan plan) { - logger.debug(String.format("Checking if storage pool [%s] is suitable to disk [%s].", pool, dskCh)); + logger.debug("Checking if storage pool [{}] is suitable to disk [{}].", pool, dskCh); if (avoid.shouldAvoid(pool)) { - logger.debug(String.format("StoragePool [%s] is in avoid set, skipping this pool to allocation of disk [%s].", pool, dskCh)); + logger.debug("StoragePool [{}] is in avoid set, skipping this pool to allocation of disk [{}].", pool, dskCh); return false; } if (dskCh.requiresEncryption() && !pool.getPoolType().supportsEncryption()) { if (logger.isDebugEnabled()) { - logger.debug(String.format("Storage pool type '%s' doesn't support encryption required for volume, skipping this pool", pool.getPoolType())); + logger.debug("Storage pool type '[{}]' doesn't support encryption required for volume, skipping this pool", pool.getPoolType()); } return false; } @@ -319,8 +313,8 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement } if (!checkDiskProvisioningSupport(dskCh, pool)) { - logger.debug(String.format("Storage pool [%s] does not have support to disk provisioning of disk [%s].", pool, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(dskCh, - "type", "name", "diskOfferingId", "templateId", "volumeId", "provisioningType", "hyperType"))); + logger.debug("Storage pool [{}] does not have support to disk provisioning of disk [{}].", pool, ReflectionToStringBuilderUtils.reflectOnlySelectedFields(dskCh, + "type", "name", "diskOfferingId", "templateId", "volumeId", "provisioningType", "hyperType")); return false; } @@ -332,7 +326,7 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement HostVO plannedHost = hostDao.findById(plan.getHostId()); if (!storageMgr.checkIfHostAndStoragePoolHasCommonStorageAccessGroups(plannedHost, pool)) { if (logger.isDebugEnabled()) { - logger.debug(String.format("StoragePool %s and host %s does not have matching storage access groups", pool, plannedHost)); + logger.debug("StoragePool [{}] and host [{}] does not have matching storage access groups", pool, plannedHost); } return false; } @@ -343,13 +337,13 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement if (!isTempVolume) { volume = volumeDao.findById(dskCh.getVolumeId()); if (!storageMgr.storagePoolCompatibleWithVolumePool(pool, volume)) { - logger.debug(String.format("Pool [%s] is not compatible with volume [%s], skipping it.", pool, volume)); + logger.debug("Pool [{}] is not compatible with volume [{}], skipping it.", pool, volume); return false; } } if (pool.isManaged() && !storageUtil.managedStoragePoolCanScale(pool, plan.getClusterId(), plan.getHostId())) { - logger.debug(String.format("Cannot allocate pool [%s] to volume [%s] because the max number of managed clustered filesystems has been exceeded.", pool, volume)); + logger.debug("Cannot allocate pool [{}] to volume [{}] because the max number of managed clustered filesystems has been exceeded.", pool, volume); return false; } @@ -358,13 +352,13 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement requestVolumeDiskProfilePairs.add(new Pair<>(volume, dskCh)); if (dskCh.getHypervisorType() == HypervisorType.VMware) { if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster && storageMgr.isStoragePoolDatastoreClusterParent(pool)) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is a parent datastore cluster.", pool, volume)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is a parent datastore cluster.", pool, volume); return false; } if (pool.getParent() != 0L) { StoragePoolVO datastoreCluster = storagePoolDao.findById(pool.getParent()); if (datastoreCluster == null || (datastoreCluster != null && datastoreCluster.getStatus() != StoragePoolStatus.Up)) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not in [%s] state.", datastoreCluster, volume, StoragePoolStatus.Up)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is not in [{}] state.", datastoreCluster, volume, StoragePoolStatus.Up); return false; } } @@ -374,11 +368,11 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement storageMgr.isStoragePoolCompliantWithStoragePolicy(dskCh.getDiskOfferingId(), pool) : storageMgr.isStoragePoolCompliantWithStoragePolicy(requestVolumeDiskProfilePairs, pool); if (!isStoragePoolStoragePolicyCompliance) { - logger.debug(String.format("Skipping allocation of pool [%s] to volume [%s] because this pool is not compliant with the storage policy required by the volume.", pool, volume)); + logger.debug("Skipping allocation of pool [{}] to volume [{}] because this pool is not compliant with the storage policy required by the volume.", pool, volume); return false; } } catch (StorageUnavailableException e) { - logger.warn(String.format("Could not verify storage policy compliance against storage pool %s due to exception %s", pool.getUuid(), e.getMessage())); + logger.warn("Could not verify storage policy compliance against storage pool [{}] due to exception [{}]", pool.getUuid(), e.getMessage()); return false; } } @@ -427,19 +421,19 @@ public abstract class AbstractStoragePoolAllocator extends AdapterBase implement protected void logDisabledStoragePools(long dcId, Long podId, Long clusterId, ScopeType scope) { List disabledPools = storagePoolDao.findDisabledPoolsByScope(dcId, podId, clusterId, scope); if (disabledPools != null && !disabledPools.isEmpty()) { - logger.trace(String.format("Ignoring pools [%s] as they are in disabled state.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(disabledPools))); + logger.trace("Ignoring pools [{}] as they are in disabled state.", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(disabledPools)); } } protected void logStartOfSearch(DiskProfile dskCh, VirtualMachineProfile vmProfile, DeploymentPlan plan, int returnUpTo, boolean bypassStorageTypeCheck){ - logger.trace(String.format("%s is looking for storage pools that match the VM's disk profile [%s], virtual machine profile [%s] and " - + "deployment plan [%s]. Returning up to [%d] and bypassStorageTypeCheck [%s].", this.getClass().getSimpleName(), dskCh, vmProfile, plan, returnUpTo, bypassStorageTypeCheck)); + logger.trace("[{}] is looking for storage pools that match the VM's disk profile [{}], virtual machine profile [{}] and " + + "deployment plan [{}]. Returning up to [{}] and bypassStorageTypeCheck [{}].", this.getClass().getSimpleName(), dskCh, vmProfile, plan, returnUpTo, bypassStorageTypeCheck); } protected void logEndOfSearch(List storagePoolList) { - logger.debug(String.format("%s is returning [%s] suitable storage pools [%s].", this.getClass().getSimpleName(), storagePoolList.size(), - Arrays.toString(storagePoolList.toArray()))); + logger.debug("[{}] is returning [{}] suitable storage pools [{}].", this.getClass().getSimpleName(), storagePoolList.size(), + Arrays.toString(storagePoolList.toArray())); } } diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java index a2e9eff2a08..26b39e30776 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java @@ -230,8 +230,10 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { updateBuilder.setJobId(answer.getJobId()); updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); updateBuilder.setInstallPath(answer.getInstallPath()); - updateBuilder.setSize(answer.getTemplateSize()); - updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + } _templateStoreDao.update(tmpltStoreVO.getId(), updateBuilder); // update size in vm_template table VMTemplateVO tmlptUpdater = _templateDao.createForUpdate(); @@ -241,8 +243,7 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { AsyncCompletionCallback caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); @@ -285,19 +286,22 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver { updateBuilder.setJobId(answer.getJobId()); updateBuilder.setLocalDownloadPath(answer.getDownloadPath()); updateBuilder.setInstallPath(answer.getInstallPath()); - updateBuilder.setSize(answer.getTemplateSize()); - updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + updateBuilder.setSize(answer.getTemplateSize()); + updateBuilder.setPhysicalSize(answer.getTemplatePhySicalSize()); + } _volumeStoreDao.update(volStoreVO.getId(), updateBuilder); // update size in volume table - VolumeVO volUpdater = volumeDao.createForUpdate(); - volUpdater.setSize(answer.getTemplateSize()); - volumeDao.update(obj.getId(), volUpdater); + if (!VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { + VolumeVO volUpdater = volumeDao.createForUpdate(); + volUpdater.setSize(answer.getTemplateSize()); + volumeDao.update(obj.getId(), volUpdater); + } } AsyncCompletionCallback caller = context.getParentCallback(); - if (answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR || - answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.ABANDONED || answer.getDownloadStatus() == VMTemplateStorageResourceAssoc.Status.UNKNOWN) { + if (VMTemplateStorageResourceAssoc.ERROR_DOWNLOAD_STATES.contains(answer.getDownloadStatus())) { CreateCmdResult result = new CreateCmdResult(null, null); result.setSuccess(false); result.setResult(answer.getErrorString()); diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 436f991afbd..8731e8791dd 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -395,9 +395,9 @@ public class VolumeServiceImpl implements VolumeService { } // Find out if the volume is at state of download_in_progress on secondary storage - VolumeDataStoreVO volumeStore = _volumeStoreDao.findByVolume(volume.getId()); - if (volumeStore != null) { - if (volumeStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { + VolumeDataStoreVO volumeOnImageStore = _volumeStoreDao.findByVolume(volume.getId()); + if (volumeOnImageStore != null) { + if (volumeOnImageStore.getDownloadState() == VMTemplateStorageResourceAssoc.Status.DOWNLOAD_IN_PROGRESS) { String msg = String.format("Volume: %s is currently being uploaded; can't delete it.", volume); logger.debug(msg); result.setSuccess(false); @@ -416,10 +416,10 @@ public class VolumeServiceImpl implements VolumeService { if (!volumeExistsOnPrimary(vol)) { // not created on primary store - if (volumeStore == null) { + if (volumeOnImageStore == null) { // also not created on secondary store if (logger.isDebugEnabled()) { - logger.debug("Marking volume that was never created as destroyed: " + vol); + logger.debug("Marking volume that was never created as destroyed: {}", vol); } VMTemplateVO template = templateDao.findById(vol.getTemplateId()); if (template != null && !template.isDeployAsIs()) { @@ -435,11 +435,21 @@ public class VolumeServiceImpl implements VolumeService { if (volume.getDataStore().getRole() == DataStoreRole.Image) { // no need to change state in volumes table volume.processEventOnly(Event.DestroyRequested); + if (volumeOnImageStore == null) { + logger.debug("Volume {} doesn't exist on image store, no need to delete", vol); + future.complete(result); + return future; + } } else if (volume.getDataStore().getRole() == DataStoreRole.Primary) { if (vol.getState() == Volume.State.Expunging) { logger.info("Volume {} is already in Expunging, retrying", volume); } volume.processEvent(Event.ExpungeRequested); + if (!volumeExistsOnPrimary(vol)) { + logger.debug("Volume {} doesn't exist on primary storage, no need to delete", vol); + future.complete(result); + return future; + } } DeleteVolumeContext context = new DeleteVolumeContext<>(null, vo, future); @@ -460,13 +470,11 @@ public class VolumeServiceImpl implements VolumeService { private boolean volumeExistsOnPrimary(VolumeVO vol) { Long poolId = vol.getPoolId(); - if (poolId == null) { return false; } PrimaryDataStore primaryStore = dataStoreMgr.getPrimaryDataStore(poolId); - if (primaryStore == null) { return false; } @@ -476,8 +484,7 @@ public class VolumeServiceImpl implements VolumeService { } String volumePath = vol.getPath(); - - if (volumePath == null || volumePath.trim().isEmpty()) { + if (StringUtils.isBlank(volumePath)) { return false; } diff --git a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java index 15807eb26d4..3323d5c4d82 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java +++ b/framework/db/src/main/java/com/cloud/utils/db/SearchCriteria.java @@ -205,6 +205,12 @@ public class SearchCriteria { } + public void setJoinParametersIfNotNull(String joinName, String conditionName, Object... params) { + if (ArrayUtils.isNotEmpty(params) && (params.length > 1 || params[0] != null)) { + setJoinParameters(joinName, conditionName, params); + } + } + public SearchCriteria getJoin(String joinName) { return _joins.get(joinName).getT(); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java index 5ab54149645..9d76a7e6ec2 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmd.java @@ -83,6 +83,12 @@ public class CreateExtensionCmd extends BaseCmd { description = "Details in key/value pairs using format details[i].keyname=keyvalue. Example: details[0].endpoint.url=urlvalue") protected Map details; + @Parameter(name = ApiConstants.RESERVED_RESOURCE_DETAILS, type = CommandType.STRING, + description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -115,6 +121,10 @@ public class CreateExtensionCmd extends BaseCmd { return convertDetailsToMap(details); } + public String getReservedResourceDetails() { + return reservedResourceDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java index ded07d2dd32..5baaea1709d 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmd.java @@ -78,6 +78,12 @@ public class UpdateExtensionCmd extends BaseCmd { "if false or not set, no action)") private Boolean cleanupDetails; + @Parameter(name = ApiConstants.RESERVED_RESOURCE_DETAILS, type = CommandType.STRING, + description = "Resource detail names as comma separated string that should be reserved and not visible " + + "to end users", + since = "4.22.1") + protected String reservedResourceDetails; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -106,6 +112,10 @@ public class UpdateExtensionCmd extends BaseCmd { return cleanupDetails; } + public String getReservedResourceDetails() { + return reservedResourceDetails; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index 4171b9615fe..f6fd08b6da2 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -216,6 +216,11 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject AccountService accountService; + // Map of in-built extension names and their reserved resource details that shouldn't be accessible to end-users + protected static final Map> INBUILT_RESERVED_RESOURCE_DETAILS = Map.of( + "proxmox", List.of("proxmox_vmid") + ); + private ScheduledExecutorService extensionPathStateCheckExecutor; protected String getDefaultExtensionRelativePath(String name) { @@ -563,6 +568,25 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana updateExtensionPathReady(extension, true); } + protected void addInbuiltExtensionReservedResourceDetails(long extensionId, List reservedResourceDetails) { + ExtensionVO vo = extensionDao.findById(extensionId); + if (vo == null || vo.isUserDefined()) { + return; + } + String lowerName = StringUtils.defaultString(vo.getName()).toLowerCase(); + Optional>> match = INBUILT_RESERVED_RESOURCE_DETAILS.entrySet().stream() + .filter(e -> lowerName.contains(e.getKey().toLowerCase())) + .findFirst(); + if (match.isPresent()) { + Set existing = new HashSet<>(reservedResourceDetails); + for (String detailKey : match.get().getValue()) { + if (existing.add(detailKey)) { + reservedResourceDetails.add(detailKey); + } + } + } + } + @Override public String getExtensionsPath() { return externalProvisioner.getExtensionsPath(); @@ -577,6 +601,7 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana String relativePath = cmd.getPath(); final Boolean orchestratorRequiresPrepareVm = cmd.isOrchestratorRequiresPrepareVm(); final String stateStr = cmd.getState(); + final String reservedResourceDetails = cmd.getReservedResourceDetails(); ExtensionVO extensionByName = extensionDao.findByName(name); if (extensionByName != null) { throw new CloudRuntimeException("Extension by name already exists"); @@ -624,6 +649,10 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, String.valueOf(orchestratorRequiresPrepareVm), false)); } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + detailsVOList.add(new ExtensionDetailsVO(extension.getId(), + ApiConstants.RESERVED_RESOURCE_DETAILS, reservedResourceDetails, false)); + } if (CollectionUtils.isNotEmpty(detailsVOList)) { extensionDetailsDao.saveDetails(detailsVOList); } @@ -704,6 +733,7 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana final String stateStr = cmd.getState(); final Map details = cmd.getDetails(); final Boolean cleanupDetails = cmd.isCleanupDetails(); + final String reservedResourceDetails = cmd.getReservedResourceDetails(); final ExtensionVO extensionVO = extensionDao.findById(id); if (extensionVO == null) { throw new InvalidParameterValueException("Failed to find the extension"); @@ -732,7 +762,8 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana throw new CloudRuntimeException(String.format("Failed to updated the extension: %s", extensionVO.getName())); } - updateExtensionsDetails(cleanupDetails, details, orchestratorRequiresPrepareVm, id); + updateExtensionsDetails(cleanupDetails, details, orchestratorRequiresPrepareVm, reservedResourceDetails, + id); return extensionVO; }); if (StringUtils.isNotBlank(stateStr)) { @@ -748,9 +779,11 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana return result; } - protected void updateExtensionsDetails(Boolean cleanupDetails, Map details, Boolean orchestratorRequiresPrepareVm, long id) { + protected void updateExtensionsDetails(Boolean cleanupDetails, Map details, + Boolean orchestratorRequiresPrepareVm, String reservedResourceDetails, long id) { final boolean needToUpdateAllDetails = Boolean.TRUE.equals(cleanupDetails) || MapUtils.isNotEmpty(details); - if (!needToUpdateAllDetails && orchestratorRequiresPrepareVm == null) { + if (!needToUpdateAllDetails && orchestratorRequiresPrepareVm == null && + StringUtils.isBlank(reservedResourceDetails)) { return; } if (needToUpdateAllDetails) { @@ -761,6 +794,9 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana hiddenDetails.put(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, String.valueOf(orchestratorRequiresPrepareVm)); } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + hiddenDetails.put(ApiConstants.RESERVED_RESOURCE_DETAILS, reservedResourceDetails); + } if (MapUtils.isNotEmpty(hiddenDetails)) { hiddenDetails.forEach((key, value) -> detailsVOList.add( new ExtensionDetailsVO(id, key, value, false))); @@ -775,15 +811,29 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana extensionDetailsDao.removeDetails(id); } } else { - ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(id, - ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM); - if (detailsVO == null) { - extensionDetailsDao.persist(new ExtensionDetailsVO(id, - ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, - String.valueOf(orchestratorRequiresPrepareVm), false)); - } else if (Boolean.parseBoolean(detailsVO.getValue()) != orchestratorRequiresPrepareVm) { - detailsVO.setValue(String.valueOf(orchestratorRequiresPrepareVm)); - extensionDetailsDao.update(detailsVO.getId(), detailsVO); + if (orchestratorRequiresPrepareVm != null) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(id, + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM); + if (detailsVO == null) { + extensionDetailsDao.persist(new ExtensionDetailsVO(id, + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, + String.valueOf(orchestratorRequiresPrepareVm), false)); + } else if (Boolean.parseBoolean(detailsVO.getValue()) != orchestratorRequiresPrepareVm) { + detailsVO.setValue(String.valueOf(orchestratorRequiresPrepareVm)); + extensionDetailsDao.update(detailsVO.getId(), detailsVO); + } + } + if (StringUtils.isNotBlank(reservedResourceDetails)) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(id, + ApiConstants.RESERVED_RESOURCE_DETAILS); + if (detailsVO == null) { + extensionDetailsDao.persist(new ExtensionDetailsVO(id, + ApiConstants.RESERVED_RESOURCE_DETAILS, + reservedResourceDetails, false)); + } else if (!reservedResourceDetails.equals(detailsVO.getValue())) { + detailsVO.setValue(reservedResourceDetails); + extensionDetailsDao.update(detailsVO.getId(), detailsVO); + } } } } @@ -961,12 +1011,16 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana hiddenDetails = extensionDetails.second(); } else { hiddenDetails = extensionDetailsDao.listDetailsKeyPairs(extension.getId(), - List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM)); + List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, + ApiConstants.RESERVED_RESOURCE_DETAILS)); } if (hiddenDetails.containsKey(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM)) { response.setOrchestratorRequiresPrepareVm(Boolean.parseBoolean( hiddenDetails.get(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM))); } + if (hiddenDetails.containsKey(ApiConstants.RESERVED_RESOURCE_DETAILS)) { + response.setReservedResourceDetails(hiddenDetails.get(ApiConstants.RESERVED_RESOURCE_DETAILS)); + } response.setObjectName(Extension.class.getSimpleName().toLowerCase()); return response; } @@ -1605,6 +1659,24 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana return extensionDao.findById(extensionId); } + @Override + public List getExtensionReservedResourceDetails(long extensionId) { + ExtensionDetailsVO detailsVO = extensionDetailsDao.findDetail(extensionId, + ApiConstants.RESERVED_RESOURCE_DETAILS); + List reservedDetails = new ArrayList<>(); + if (detailsVO != null && StringUtils.isNotBlank(detailsVO.getValue())) { + String[] parts = detailsVO.getValue().split(","); + for (String part : parts) { + String trimmedPart = part.trim(); + if (StringUtils.isNotBlank(trimmedPart)) { + reservedDetails.add(trimmedPart); + } + } + } + addInbuiltExtensionReservedResourceDetails(extensionId, reservedDetails); + return reservedDetails; + } + @Override public boolean start() { long pathStateCheckInterval = PathStateCheckInterval.value(); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java index 2edb6ea48e3..2f630966056 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java @@ -94,4 +94,18 @@ public class CreateExtensionCmdTest { setField(cmd, "details", details); assertTrue(MapUtils.isNotEmpty(cmd.getDetails())); } + + @Test + public void getReservedResourceDetailsReturnsValueWhenSet() { + setField(cmd, "reservedResourceDetails", "detail1,detail2,detail3"); + String result = cmd.getReservedResourceDetails(); + assertEquals("detail1,detail2,detail3", result); + } + + @Test + public void getReservedResourceDetailsReturnsNullWhenNotSet() { + setField(cmd, "reservedResourceDetails", null); + String result = cmd.getReservedResourceDetails(); + assertNull(result); + } } diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java index f0a3a6fcf21..5c5c2014a52 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.springframework.test.util.ReflectionTestUtils.setField; import java.util.EnumSet; import java.util.HashMap; @@ -134,6 +135,20 @@ public class UpdateExtensionCmdTest { assertTrue(cmd.isCleanupDetails()); } + @Test + public void getReservedResourceDetailsReturnsValueWhenSet() { + setField(cmd, "reservedResourceDetails", "detail1,detail2,detail3"); + String result = cmd.getReservedResourceDetails(); + assertEquals("detail1,detail2,detail3", result); + } + + @Test + public void getReservedResourceDetailsReturnsNullWhenNotSet() { + setField(cmd, "reservedResourceDetails", null); + String result = cmd.getReservedResourceDetails(); + assertNull(result); + } + @Test public void executeSetsExtensionResponseWhenManagerSucceeds() { Extension extension = mock(Extension.class); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java index 085ae212b28..ff3fce06b00 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImplTest.java @@ -23,11 +23,13 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.Mockito.any; import static org.mockito.Mockito.anyBoolean; import static org.mockito.Mockito.anyLong; import static org.mockito.Mockito.anyString; import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; @@ -40,6 +42,7 @@ import static org.mockito.Mockito.when; import java.io.File; import java.security.InvalidParameterException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -49,8 +52,6 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.user.AccountService; import org.apache.cloudstack.acl.Role; import org.apache.cloudstack.acl.RoleService; import org.apache.cloudstack.acl.RoleType; @@ -85,9 +86,11 @@ import org.apache.cloudstack.framework.extensions.dao.ExtensionResourceMapDao; import org.apache.cloudstack.framework.extensions.dao.ExtensionResourceMapDetailsDao; import org.apache.cloudstack.framework.extensions.vo.ExtensionCustomActionDetailsVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionCustomActionVO; +import org.apache.cloudstack.framework.extensions.vo.ExtensionDetailsVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionResourceMapVO; import org.apache.cloudstack.framework.extensions.vo.ExtensionVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; +import org.apache.commons.collections.CollectionUtils; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -113,6 +116,7 @@ import com.cloud.dc.dao.ClusterDao; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.exception.PermissionDeniedException; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; @@ -122,6 +126,7 @@ import com.cloud.org.Cluster; import com.cloud.serializer.GsonHelper; import com.cloud.storage.dao.VMTemplateDao; import com.cloud.user.Account; +import com.cloud.user.AccountService; import com.cloud.utils.Pair; import com.cloud.utils.UuidUtils; import com.cloud.utils.db.EntityManager; @@ -664,6 +669,8 @@ public class ExtensionsManagerImplTest { when(cmd.getPath()).thenReturn(null); when(cmd.isOrchestratorRequiresPrepareVm()).thenReturn(null); when(cmd.getState()).thenReturn(null); + String reservedResourceDetails = "abc,xyz"; + when(cmd.getReservedResourceDetails()).thenReturn(reservedResourceDetails); when(extensionDao.findByName("ext1")).thenReturn(null); when(extensionDao.persist(any())).thenAnswer(inv -> { ExtensionVO extensionVO = inv.getArgument(0); @@ -671,11 +678,20 @@ public class ExtensionsManagerImplTest { return extensionVO; }); when(managementServerHostDao.listBy(any())).thenReturn(Collections.emptyList()); - + List detailsList = new ArrayList<>(); + doAnswer(inv -> { + List detailsVO = inv.getArgument(0); + detailsList.addAll(detailsVO); + return null; + }).when(extensionDetailsDao).saveDetails(anyList()); Extension ext = extensionsManager.createExtension(cmd); assertEquals("ext1", ext.getName()); verify(extensionDao).persist(any()); + assertTrue(CollectionUtils.isNotEmpty(detailsList)); + assertTrue(detailsList.stream() + .anyMatch(detail -> ApiConstants.RESERVED_RESOURCE_DETAILS.equals(detail.getName()) + && reservedResourceDetails.equals(detail.getValue()))); } @Test @@ -938,14 +954,32 @@ public class ExtensionsManagerImplTest { public void updateExtensionsDetails_SavesDetails_WhenDetailsProvided() { long extensionId = 10L; Map details = Map.of("foo", "bar", "baz", "qux"); - extensionsManager.updateExtensionsDetails(false, details, null, extensionId); + extensionsManager.updateExtensionsDetails(false, details, null, null, extensionId); verify(extensionDetailsDao).saveDetails(any()); } + @Test + public void updateExtensionsDetails_PersistReservedDetail_WhenProvided() { + long extensionId = 10L; + when(extensionDetailsDao.persist(any())).thenReturn(mock(ExtensionDetailsVO.class)); + extensionsManager.updateExtensionsDetails(false, null, null, "abc,xyz", extensionId); + verify(extensionDetailsDao).persist(any()); + } + + @Test + public void updateExtensionsDetails_UpdateReservedDetail_WhenProvided() { + long extensionId = 10L; + when(extensionDetailsDao.findDetail(anyLong(), eq(ApiConstants.RESERVED_RESOURCE_DETAILS))) + .thenReturn(mock(ExtensionDetailsVO.class)); + when(extensionDetailsDao.update(anyLong(), any())).thenReturn(true); + extensionsManager.updateExtensionsDetails(false, null, null, "abc,xyz", extensionId); + verify(extensionDetailsDao).update(anyLong(), any()); + } + @Test public void updateExtensionsDetails_DoesNothing_WhenDetailsAndCleanupAreNull() { long extensionId = 11L; - extensionsManager.updateExtensionsDetails(null, null, null, extensionId); + extensionsManager.updateExtensionsDetails(null, null, null, null, extensionId); verify(extensionDetailsDao, never()).removeDetails(anyLong()); verify(extensionDetailsDao, never()).saveDetails(any()); } @@ -953,7 +987,7 @@ public class ExtensionsManagerImplTest { @Test public void updateExtensionsDetails_RemovesDetailsOnly_WhenCleanupIsTrue() { long extensionId = 12L; - extensionsManager.updateExtensionsDetails(true, null, null, extensionId); + extensionsManager.updateExtensionsDetails(true, null, null, null, extensionId); verify(extensionDetailsDao).removeDetails(extensionId); verify(extensionDetailsDao, never()).saveDetails(any()); } @@ -961,7 +995,7 @@ public class ExtensionsManagerImplTest { @Test public void updateExtensionsDetails_PersistsOrchestratorFlag_WhenFlagIsNotNull() { long extensionId = 13L; - extensionsManager.updateExtensionsDetails(false, null, true, extensionId); + extensionsManager.updateExtensionsDetails(false, null, true, null, extensionId); verify(extensionDetailsDao).persist(any()); } @@ -970,7 +1004,7 @@ public class ExtensionsManagerImplTest { long extensionId = 14L; Map details = Map.of("foo", "bar"); doThrow(CloudRuntimeException.class).when(extensionDetailsDao).saveDetails(any()); - extensionsManager.updateExtensionsDetails(false, details, null, extensionId); + extensionsManager.updateExtensionsDetails(false, details, null, null, extensionId); } @Test @@ -1161,7 +1195,8 @@ public class ExtensionsManagerImplTest { when(externalProvisioner.getExtensionPath("entry2.sh")).thenReturn("/some/path/entry2.sh"); Map hiddenDetails = Map.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, "false"); - when(extensionDetailsDao.listDetailsKeyPairs(2L, List.of(ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM))) + when(extensionDetailsDao.listDetailsKeyPairs(2L, List.of( + ApiConstants.ORCHESTRATOR_REQUIRES_PREPARE_VM, ApiConstants.RESERVED_RESOURCE_DETAILS))) .thenReturn(hiddenDetails); EnumSet viewDetails = EnumSet.noneOf(ApiConstants.ExtensionDetails.class); @@ -2069,4 +2104,118 @@ public class ExtensionsManagerImplTest { } } + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenDetailsNotFound() { + long extensionId = 1L; + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(null); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenValueIsBlank() { + long extensionId = 2L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(" "); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsListOfTrimmedDetails() { + long extensionId = 3L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(" detail1 , detail2,detail3 "); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("detail1", result.get(0)); + assertEquals("detail2", result.get(1)); + assertEquals("detail3", result.get(2)); + } + + @Test + public void getExtensionReservedResourceDetailsHandlesEmptyPartsGracefully() { + long extensionId = 4L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn("detail1,,detail2, ,detail3"); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("detail1", result.get(0)); + assertEquals("detail2", result.get(1)); + assertEquals("detail3", result.get(2)); + } + + @Test + public void getExtensionReservedResourceDetailsReturnsEmptyListWhenSplitResultsInNoParts() { + long extensionId = 5L; + ExtensionDetailsVO detailsVO = mock(ExtensionDetailsVO.class); + when(detailsVO.getValue()).thenReturn(","); + when(extensionDetailsDao.findDetail(extensionId, ApiConstants.RESERVED_RESOURCE_DETAILS)).thenReturn(detailsVO); + + List result = extensionsManager.getExtensionReservedResourceDetails(extensionId); + + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingWhenExtensionNotFound() { + when(extensionDao.findById(1L)).thenReturn(null); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(1L, reservedResourceDetails); + assertTrue(reservedResourceDetails.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingForUserDefinedExtension() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(true); + when(extensionDao.findById(2L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + reservedResourceDetails.add("existing-detail"); + extensionsManager.addInbuiltExtensionReservedResourceDetails(2L, reservedResourceDetails); + assertEquals(1, reservedResourceDetails.size()); + assertTrue(reservedResourceDetails.contains("existing-detail")); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsDoesNothingWhenNoMatchFound() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(false); + when(extension.getName()).thenReturn("no-such-inbuilt-key-expected"); + when(extensionDao.findById(3L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(3L, reservedResourceDetails); + assertTrue(reservedResourceDetails.isEmpty()); + } + + @Test + public void addInbuiltExtensionReservedResourceDetailsAddedDetails() { + ExtensionVO extension = mock(ExtensionVO.class); + when(extension.isUserDefined()).thenReturn(false); + Map.Entry> entry = + ExtensionsManagerImpl.INBUILT_RESERVED_RESOURCE_DETAILS.entrySet().iterator().next(); + when(extension.getName()).thenReturn(entry.getKey()); + when(extensionDao.findById(3L)).thenReturn(extension); + List reservedResourceDetails = new ArrayList<>(); + extensionsManager.addInbuiltExtensionReservedResourceDetails(3L, reservedResourceDetails); + assertFalse(reservedResourceDetails.isEmpty()); + assertEquals(reservedResourceDetails.size(), entry.getValue().size()); + assertTrue(reservedResourceDetails.containsAll(entry.getValue())); + } } diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java index 9f7a4ad6e05..926280bfead 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDao.java @@ -23,12 +23,30 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import com.cloud.utils.db.GenericDao; +import javax.annotation.Nullable; + public interface AsyncJobDao extends GenericDao { AsyncJobVO findInstancePendingAsyncJob(String instanceType, long instanceId); List findInstancePendingAsyncJobs(String instanceType, Long accountId); + /** + * Finds async job matching the given parameters. + * Non-null parameters are added to search criteria. + * Returns the most recent job by creation date. + *

+ * When searching by resourceId and resourceType, only one active job + * is expected per resource, so returning a single result is sufficient. + * + * @param id job ID + * @param resourceId resource ID (instanceId) + * @param resourceType resource type (instanceType) + * @return matching job or null + */ + @Nullable + AsyncJobVO findJob(Long id, Long resourceId, String resourceType); + AsyncJobVO findPseudoJob(long threadId, long msid); void cleanupPseduoJobs(long msid); diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java index a2f1f36b863..81cc5d4f2a8 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/dao/AsyncJobDaoImpl.java @@ -22,6 +22,8 @@ import java.util.Date; import java.util.List; import org.apache.cloudstack.api.ApiConstants; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.jobs.JobInfo; @@ -45,6 +47,7 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements private final SearchBuilder expiringUnfinishedAsyncJobSearch; private final SearchBuilder expiringCompletedAsyncJobSearch; private final SearchBuilder failureMsidAsyncJobSearch; + private final SearchBuilder byIdResourceIdResourceTypeSearch; private final GenericSearchBuilder asyncJobTypeSearch; private final GenericSearchBuilder pendingNonPseudoAsyncJobsSearch; @@ -95,6 +98,12 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements failureMsidAsyncJobSearch.and("job_cmd", failureMsidAsyncJobSearch.entity().getCmd(), Op.IN); failureMsidAsyncJobSearch.done(); + byIdResourceIdResourceTypeSearch = createSearchBuilder(); + byIdResourceIdResourceTypeSearch.and("id", byIdResourceIdResourceTypeSearch.entity().getId(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.and("instanceId", byIdResourceIdResourceTypeSearch.entity().getInstanceId(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.and("instanceType", byIdResourceIdResourceTypeSearch.entity().getInstanceType(), SearchCriteria.Op.EQ); + byIdResourceIdResourceTypeSearch.done(); + asyncJobTypeSearch = createSearchBuilder(Long.class); asyncJobTypeSearch.select(null, SearchCriteria.Func.COUNT, asyncJobTypeSearch.entity().getId()); asyncJobTypeSearch.and("job_info", asyncJobTypeSearch.entity().getCmdInfo(),Op.LIKE); @@ -140,6 +149,30 @@ public class AsyncJobDaoImpl extends GenericDaoBase implements return listBy(sc); } + @Override + public AsyncJobVO findJob(Long id, Long resourceId, String resourceType) { + SearchCriteria sc = byIdResourceIdResourceTypeSearch.create(); + + if (id == null && resourceId == null && StringUtils.isBlank(resourceType)) { + logger.debug("findJob called with all null parameters"); + return null; + } + + if (id != null) { + sc.setParameters("id", id); + } + if (resourceId != null && StringUtils.isNotBlank(resourceType)) { + sc.setParameters("instanceType", resourceType); + sc.setParameters("instanceId", resourceId); + } + Filter filter = new Filter(AsyncJobVO.class, "created", false, 0L, 1L); + List result = searchIncludingRemoved(sc, filter, Boolean.FALSE, false); + if (CollectionUtils.isNotEmpty(result)) { + return result.get(0); + } + return null; + } + @Override public AsyncJobVO findPseudoJob(long threadId, long msid) { SearchCriteria sc = pseudoJobSearch.create(); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java index 816144aa2f1..58f51eae9fc 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/QuotaManagerImpl.java @@ -150,8 +150,9 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { return; } - Date startDate = accountQuotaUsages.get(0).getStartDate(); - Date endDate = accountQuotaUsages.get(0).getEndDate(); + QuotaUsageVO firstQuotaUsage = accountQuotaUsages.get(0); + Date startDate = firstQuotaUsage.getStartDate(); + Date endDate = firstQuotaUsage.getEndDate(); Date lastQuotaUsageEndDate = accountQuotaUsages.get(accountQuotaUsages.size() - 1).getEndDate(); LinkedHashSet> periods = accountQuotaUsages.stream() @@ -215,7 +216,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { logger.debug(String.format("Persisting the first quota balance [%s] for account [%s].", firstBalance, accountToString)); _quotaBalanceDao.saveQuotaBalance(firstBalance); } else { - QuotaBalanceVO lastRealBalance = _quotaBalanceDao.findLastBalanceEntry(accountId, domainId, startDate); + QuotaBalanceVO lastRealBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, startDate); if (lastRealBalance == null) { logger.warn("Account [{}] has quota usage entries, however it does not have a quota balance.", accountToString); @@ -244,7 +245,7 @@ public class QuotaManagerImpl extends ManagerBase implements QuotaManager { } protected BigDecimal aggregateCreditBetweenDates(Long accountId, Long domainId, Date startDate, Date endDate, String accountToString) { - List creditsReceived = _quotaBalanceDao.findCreditBalance(accountId, domainId, startDate, endDate); + List creditsReceived = _quotaBalanceDao.findCreditBalances(accountId, domainId, startDate, endDate); logger.debug("Account [{}] has [{}] credit entries before [{}].", accountToString, creditsReceived.size(), DateUtil.displayDateInTimezone(usageAggregationTimeZone, endDate)); diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index a2fca7b80fd..23020292027 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -28,12 +28,14 @@ import java.util.stream.Collectors; import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; +import com.cloud.hypervisor.Hypervisor; import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.VpcOfferingVO; import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; -import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; @@ -66,6 +68,7 @@ import com.cloud.domain.dao.DomainDao; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostTagsDao; +import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.offerings.NetworkOfferingVO; import com.cloud.offerings.dao.NetworkOfferingDao; import com.cloud.server.ResourceTag; @@ -191,6 +194,9 @@ public class PresetVariableHelper { @Inject ClusterDetailsDao clusterDetailsDao; + @Inject + VpcOfferingDao vpcOfferingDao; + protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value(); private List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); @@ -778,6 +784,30 @@ public class PresetVariableHelper { value.setId(network.getUuid()); value.setName(network.getName()); value.setState(usageRecord.getState()); + value.setResourceCounting(getPresetVariableValueNetworkResourceCounting(networkId)); + value.setNetworkOffering(getPresetVariableValueNetworkOffering(network.getNetworkOfferingId())); + } + + protected ResourceCounting getPresetVariableValueNetworkResourceCounting(Long networkId) { + ResourceCounting resourceCounting = new ResourceCounting(); + List vmInstancesVO = vmInstanceDao.listNonRemovedVmsByTypeAndNetwork(networkId, VirtualMachine.Type.User); + int runningVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Running)).count(); + int stoppedVms = (int) vmInstancesVO.stream().filter(vm -> vm.getState().equals(VirtualMachine.State.Stopped)).count(); + + resourceCounting.setRunningVms(runningVms); + resourceCounting.setStoppedVms(stoppedVms); + return resourceCounting; + } + + protected GenericPresetVariable getPresetVariableValueNetworkOffering(Long networkOfferingId) { + NetworkOfferingVO networkOfferingVo = networkOfferingDao.findByIdIncludingRemoved(networkOfferingId); + validateIfObjectIsNull(networkOfferingVo, networkOfferingId, "network offering"); + + GenericPresetVariable networkOffering = new GenericPresetVariable(); + networkOffering.setId(networkOfferingVo.getUuid()); + networkOffering.setName(networkOfferingVo.getName()); + + return networkOffering; } protected void loadPresetVariableValueForVpc(UsageVO usageRecord, Value value) { @@ -793,6 +823,18 @@ public class PresetVariableHelper { value.setId(vpc.getUuid()); value.setName(vpc.getName()); + value.setVpcOffering(getPresetVariableValueVpcOffering(vpc.getVpcOfferingId())); + } + + protected GenericPresetVariable getPresetVariableValueVpcOffering(Long vpcOfferingId) { + VpcOfferingVO vpcOfferingVo = vpcOfferingDao.findByIdIncludingRemoved(vpcOfferingId); + validateIfObjectIsNull(vpcOfferingVo, vpcOfferingId, "vpc offering"); + + GenericPresetVariable vpcOffering = new GenericPresetVariable(); + vpcOffering.setId(vpcOfferingVo.getUuid()); + vpcOffering.setName(vpcOfferingVo.getName()); + + return vpcOffering; } /** diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java new file mode 100644 index 00000000000..75049c3486a --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/ResourceCounting.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.quota.activationrule.presetvariables; + + +import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class ResourceCounting { + + @PresetVariableDefinition(description = "The number of running user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int runningVms; + @PresetVariableDefinition(description = "The number of stopped user instances.", supportedTypes = {QuotaTypes.NETWORK}) + private int stoppedVms; + + public int getRunningVms() { + return runningVms; + } + + public void setRunningVms(int runningVms) { + this.runningVms = runningVms; + } + + public int getStoppedVms() { + return stoppedVms; + } + + public void setStoppedVms(int stoppedVms) { + this.stoppedVms = stoppedVms; + } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java index ac776d13c57..286fe5c60fd 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/Value.java @@ -84,6 +84,9 @@ public class Value extends GenericPresetVariable { @PresetVariableDefinition(description = "Backup offering of the backup.", supportedTypes = {QuotaTypes.BACKUP}) private BackupOffering backupOffering; + @PresetVariableDefinition(description = "The amount of resources of the usage type.") + private ResourceCounting resourceCounting; + @PresetVariableDefinition(description = "The hypervisor where the resource was deployed. Values can be: XenServer, KVM, VMware, Hyperv, BareMetal, Ovm, Ovm3 and LXC.", supportedTypes = {QuotaTypes.RUNNING_VM, QuotaTypes.ALLOCATED_VM, QuotaTypes.VM_SNAPSHOT, QuotaTypes.SNAPSHOT}) private String hypervisorType; @@ -96,6 +99,12 @@ public class Value extends GenericPresetVariable { private String state; + @PresetVariableDefinition(description = "Network offering of the network.", supportedTypes = {QuotaTypes.NETWORK}) + private GenericPresetVariable networkOffering; + + @PresetVariableDefinition(description = "VPC offering of the VPC.", supportedTypes = {QuotaTypes.VPC}) + private GenericPresetVariable vpcOffering; + public Host getHost() { return host; } @@ -255,4 +264,28 @@ public class Value extends GenericPresetVariable { public void setState(String state) { this.state = state; } + + public ResourceCounting getResourceCounting() { + return resourceCounting; + } + + public void setResourceCounting(ResourceCounting resourceCounting) { + this.resourceCounting = resourceCounting; + } + + public GenericPresetVariable getNetworkOffering() { + return networkOffering; + } + + public void setNetworkOffering(GenericPresetVariable networkOffering) { + this.networkOffering = networkOffering; + } + + public GenericPresetVariable getVpcOffering() { + return vpcOffering; + } + + public void setVpcOffering(GenericPresetVariable vpcOffering) { + this.vpcOffering = vpcOffering; + } } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java index c694eaeefbe..2964746bdf0 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDao.java @@ -28,16 +28,14 @@ public interface QuotaBalanceDao extends GenericDao { QuotaBalanceVO saveQuotaBalance(QuotaBalanceVO qb); - List findCreditBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List findCreditBalances(Long accountId, Long domainId, Date startDate, Date endDate); - QuotaBalanceVO findLastBalanceEntry(Long accountId, Long domainId, Date beforeThis); + QuotaBalanceVO getLastQuotaBalanceEntry(Long accountId, Long domainId, Date beforeThis); QuotaBalanceVO findLaterBalanceEntry(Long accountId, Long domainId, Date afterThis); - List findQuotaBalance(Long accountId, Long domainId, Date startDate, Date endDate); + List listQuotaBalances(Long accountId, Long domainId, Date startDate, Date endDate); - List lastQuotaBalanceVO(Long accountId, Long domainId, Date startDate); - - BigDecimal lastQuotaBalance(Long accountId, Long domainId, Date startDate); + BigDecimal getLastQuotaBalance(Long accountId, Long domainId); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java index 01272d1a618..21553ebd27b 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImpl.java @@ -18,11 +18,14 @@ package org.apache.cloudstack.quota.dao; import java.math.BigDecimal; import java.util.ArrayList; -import java.util.Collections; import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import com.cloud.utils.db.Filter; @@ -32,160 +35,104 @@ import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.Transaction; import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.db.TransactionStatus; @Component public class QuotaBalanceDaoImpl extends GenericDaoBase implements QuotaBalanceDao { + private static final Logger logger = LogManager.getLogger(QuotaBalanceDaoImpl.class); @Override - public QuotaBalanceVO findLastBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + public QuotaBalanceVO getLastQuotaBalanceEntry(final Long accountId, final Long domainId, final Date beforeThis) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + + if (beforeThis != null) { qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LT, beforeThis); - quotaBalanceEntries = search(qb.create(), filter); - return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; } + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO findLaterBalanceEntry(final Long accountId, final Long domainId, final Date afterThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - List quotaBalanceEntries = new ArrayList<>(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); - quotaBalanceEntries = search(qb.create(), filter); - return quotaBalanceEntries.size() > 0 ? quotaBalanceEntries.get(0) : null; - } + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> { + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, 1L); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GT, afterThis); + + List quotaBalanceEntries = search(qb.create(), filter); + return !quotaBalanceEntries.isEmpty() ? quotaBalanceEntries.get(0) : null; }); } @Override public QuotaBalanceVO saveQuotaBalance(final QuotaBalanceVO qb) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback() { - @Override - public QuotaBalanceVO doInTransaction(final TransactionStatus status) { - return persist(qb); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(qb)); + } + + @Override + public List findCreditBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + if (ObjectUtils.anyNull(startDate, endDate) || startDate.after(endDate)) { + return new ArrayList<>(); } + + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); + + return search(qb.create(), filter); }); } @Override - public List findCreditBalance(final Long accountId, final Long domainId, final Date lastbalancedate, final Date beforeThis) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - if ((lastbalancedate != null) && (beforeThis != null) && lastbalancedate.before(beforeThis)) { - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true, 0L, Long.MAX_VALUE); - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.GT, 0); - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, lastbalancedate, beforeThis); - return search(qb.create(), filter); - } else { - return new ArrayList(); - } + public List listQuotaBalances(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> { + QueryBuilder qb = getQuotaBalanceQueryBuilder(accountId, domainId); + qb.and(qb.entity().getCreditsId(), SearchCriteria.Op.EQ, 0); + if (startDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.GTEQ, startDate); } + if (endDate != null) { + qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, endDate); + } + + Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", true); + return listBy(qb.create(), filter); }); } @Override - public List findQuotaBalance(final Long accountId, final Long domainId, final Date startDate, final Date endDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((startDate != null) && (endDate != null) && startDate.before(endDate)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.BETWEEN, startDate, endDate); - } else { - return Collections. emptyList(); - } - quotaUsageRecords = listBy(qb.create()); - if (quotaUsageRecords.size() == 0) { - quotaUsageRecords.addAll(lastQuotaBalanceVO(accountId, domainId, startDate)); - } - return quotaUsageRecords; - - } - }); - - } - - @Override - public List lastQuotaBalanceVO(final Long accountId, final Long domainId, final Date pivotDate) { - return Transaction.execute(TransactionLegacy.USAGE_DB, new TransactionCallback>() { - @Override - public List doInTransaction(final TransactionStatus status) { - List quotaUsageRecords = null; - List trimmedRecords = new ArrayList(); - Filter filter = new Filter(QuotaBalanceVO.class, "updatedOn", false, 0L, 100L); - // ASSUMPTION there will be less than 100 continuous credit - // transactions - QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); - if (accountId != null) { - qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); - } - if (domainId != null) { - qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); - } - if ((pivotDate != null)) { - qb.and(qb.entity().getUpdatedOn(), SearchCriteria.Op.LTEQ, pivotDate); - } - quotaUsageRecords = search(qb.create(), filter); - - // get records before startDate to find start balance - for (QuotaBalanceVO entry : quotaUsageRecords) { - if (logger.isDebugEnabled()) { - logger.debug("FindQuotaBalance Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - trimmedRecords.add(entry); - } else { - trimmedRecords.add(entry); - break; // add only consecutive credit entries and last balance entry - } - } - return trimmedRecords; - } - }); - } - - @Override - public BigDecimal lastQuotaBalance(final Long accountId, final Long domainId, Date startDate) { - List quotaBalance = lastQuotaBalanceVO(accountId, domainId, startDate); - BigDecimal finalBalance = new BigDecimal(0); - if (quotaBalance.isEmpty()) { - logger.info("There are no balance entries on or before the requested date."); - return finalBalance; + public BigDecimal getLastQuotaBalance(final Long accountId, final Long domainId) { + QuotaBalanceVO quotaBalance = getLastQuotaBalanceEntry(accountId, domainId, null); + BigDecimal finalBalance = BigDecimal.ZERO; + Date startDate = DateUtils.addDays(new Date(), -1); + if (quotaBalance == null) { + logger.info("There are no balance entries for account [{}] and domain [{}]. Considering only new added credits.", accountId, domainId); + } else { + finalBalance = quotaBalance.getCreditBalance(); + startDate = quotaBalance.getUpdatedOn(); } - for (QuotaBalanceVO entry : quotaBalance) { - if (logger.isDebugEnabled()) { - logger.debug("lastQuotaBalance Entry=" + entry); - } - finalBalance = finalBalance.add(entry.getCreditBalance()); + + List credits = findCreditBalances(accountId, domainId, startDate, new Date()); + + for (QuotaBalanceVO credit : credits) { + finalBalance = finalBalance.add(credit.getCreditBalance()); } + return finalBalance; } + private QueryBuilder getQuotaBalanceQueryBuilder(Long accountId, Long domainId) { + QueryBuilder qb = QueryBuilder.create(QuotaBalanceVO.class); + qb.and(qb.entity().getAccountId(), SearchCriteria.Op.EQ, accountId); + qb.and(qb.entity().getDomainId(), SearchCriteria.Op.EQ, domainId); + return qb; + } + } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java new file mode 100644 index 00000000000..9684ca117b5 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDao.java @@ -0,0 +1,29 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import java.util.List; + +public interface QuotaTariffUsageDao extends GenericDao { + + void persistQuotaTariffUsage(QuotaTariffUsageVO quotaTariffUsage); + + List listQuotaTariffUsages(Long quotaUsageId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java new file mode 100644 index 00000000000..556f552fed6 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaTariffUsageDaoImpl.java @@ -0,0 +1,56 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.springframework.stereotype.Component; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; + +import javax.annotation.PostConstruct; +import java.util.List; + +@Component +public class QuotaTariffUsageDaoImpl extends GenericDaoBase implements QuotaTariffUsageDao { + private SearchBuilder searchQuotaTariffUsages; + + @PostConstruct + public void init() { + searchQuotaTariffUsages = createSearchBuilder(); + searchQuotaTariffUsages.and("quotaUsageId", searchQuotaTariffUsages.entity().getQuotaUsageId(), SearchCriteria.Op.EQ); + searchQuotaTariffUsages.done(); + } + + @Override + public void persistQuotaTariffUsage(final QuotaTariffUsageVO quotaTariffUsage) { + logger.trace("Persisting quota tariff usage [{}].", quotaTariffUsage); + Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback) status -> persist(quotaTariffUsage)); + } + + @Override + public List listQuotaTariffUsages(Long quotaUsageId) { + SearchCriteria sc = searchQuotaTariffUsages.create(); + sc.setParameters("quotaUsageId", quotaUsageId); + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java new file mode 100644 index 00000000000..126fa11413f --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDao.java @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; + +import java.util.Date; +import java.util.List; + +public interface QuotaUsageJoinDao extends GenericDao { + + List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId); + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java new file mode 100644 index 00000000000..b98ea2b3a5d --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/QuotaUsageJoinDaoImpl.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.quota.dao; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.JoinBuilder; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.cloudstack.quota.vo.QuotaTariffUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.commons.lang3.ObjectUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.util.Date; +import java.util.List; + +@Component +public class QuotaUsageJoinDaoImpl extends GenericDaoBase implements QuotaUsageJoinDao { + + private SearchBuilder searchQuotaUsages; + + private SearchBuilder searchQuotaUsagesJoinTariffUsages; + + @Inject + private QuotaTariffUsageDao quotaTariffUsageDao; + + @PostConstruct + public void init() { + searchQuotaUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsages); + searchQuotaUsages.done(); + + SearchBuilder searchQuotaTariffUsages = quotaTariffUsageDao.createSearchBuilder(); + searchQuotaTariffUsages.and("tariffId", searchQuotaTariffUsages.entity().getTariffId(), SearchCriteria.Op.EQ); + searchQuotaUsagesJoinTariffUsages = createSearchBuilder(); + prepareQuotaUsageSearchBuilder(searchQuotaUsagesJoinTariffUsages); + searchQuotaUsagesJoinTariffUsages.join("searchQuotaTariffUsages", searchQuotaTariffUsages, searchQuotaUsagesJoinTariffUsages.entity().getId(), + searchQuotaTariffUsages.entity().getQuotaUsageId(), JoinBuilder.JoinType.INNER); + searchQuotaUsagesJoinTariffUsages.done(); + } + + private void prepareQuotaUsageSearchBuilder(SearchBuilder searchBuilder) { + searchBuilder.and("accountId", searchBuilder.entity().getAccountId(), SearchCriteria.Op.EQ); + searchBuilder.and("domainId", searchBuilder.entity().getDomainId(), SearchCriteria.Op.EQ); + searchBuilder.and("usageType", searchBuilder.entity().getUsageType(), SearchCriteria.Op.EQ); + searchBuilder.and("resourceId", searchBuilder.entity().getResourceId(), SearchCriteria.Op.EQ); + searchBuilder.and("networkId", searchBuilder.entity().getNetworkId(), SearchCriteria.Op.EQ); + searchBuilder.and("offeringId", searchBuilder.entity().getOfferingId(), SearchCriteria.Op.EQ); + searchBuilder.and("startDate", searchBuilder.entity().getStartDate(), SearchCriteria.Op.BETWEEN); + searchBuilder.and("endDate", searchBuilder.entity().getEndDate(), SearchCriteria.Op.BETWEEN); + } + + @Override + public List findQuotaUsage(Long accountId, Long domainId, Integer usageType, Long resourceId, Long networkId, Long offeringId, Date startDate, Date endDate, Long tariffId) { + SearchCriteria sc = tariffId == null ? searchQuotaUsages.create() : searchQuotaUsagesJoinTariffUsages.create(); + + sc.setParametersIfNotNull("accountId", accountId); + sc.setParametersIfNotNull("domainId", domainId); + sc.setParametersIfNotNull("usageType", usageType); + sc.setParametersIfNotNull("resourceId", resourceId); + sc.setParametersIfNotNull("networkId", networkId); + sc.setParametersIfNotNull("offeringId", offeringId); + + if (ObjectUtils.allNotNull(startDate, endDate)) { + sc.setParameters("startDate", startDate, endDate); + sc.setParameters("endDate", startDate, endDate); + } + + sc.setJoinParametersIfNotNull("searchQuotaTariffUsages", "tariffId", tariffId); + + return Transaction.execute(TransactionLegacy.USAGE_DB, (TransactionCallback>) status -> listBy(sc)); + } + +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java new file mode 100644 index 00000000000..4fa9e771713 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaTariffUsageVO.java @@ -0,0 +1,86 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import java.math.BigDecimal; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +@Entity +@Table(name = "quota_tariff_usage") +public class QuotaTariffUsageVO implements InternalIdentity { + @Id + @Column(name = "id") + private Long id; + + @Column(name = "tariff_id") + private Long tariffId; + + @Column(name = "quota_usage_id") + private Long quotaUsageId; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + public QuotaTariffUsageVO() { + quotaUsed = new BigDecimal(0); + } + + @Override + public long getId() { + return id; + } + + public Long getTariffId() { + return tariffId; + } + + public Long getQuotaUsageId() { + return quotaUsageId; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setId(Long id) { + this.id = id; + } + + public void setTariffId(Long tariffId) { + this.tariffId = tariffId; + } + + public void setQuotaUsageId(Long quotaUsageId) { + this.quotaUsageId = quotaUsageId; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + @Override + public String toString() { + return new ReflectionToStringBuilder(this, ToStringStyle.JSON_STYLE).toString(); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java new file mode 100644 index 00000000000..df9577e23c3 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageJoinVO.java @@ -0,0 +1,179 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.quota.vo; + +import org.apache.cloudstack.api.InternalIdentity; +import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import java.math.BigDecimal; +import java.util.Date; + +@Entity +@Table(name = "quota_usage_view") +public class QuotaUsageJoinVO implements InternalIdentity { + + @Id + @Column(name = "id", updatable = false, nullable = false) + private Long id; + + @Column(name = "zone_id") + private Long zoneId = null; + + @Column(name = "account_id") + private Long accountId = null; + + @Column(name = "domain_id") + private Long domainId = null; + + @Column(name = "usage_item_id") + private Long usageItemId; + + @Column(name = "usage_type") + private int usageType; + + @Column(name = "quota_used") + private BigDecimal quotaUsed; + + @Column(name = "start_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date startDate = null; + + @Column(name = "end_date") + @Temporal(value = TemporalType.TIMESTAMP) + private Date endDate = null; + + @Column(name = "resource_id") + private Long resourceId = null; + + @Column(name = "network_id") + private Long networkId = null; + + @Column(name = "offering_id") + private Long offeringId = null; + + @Override + public long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getZoneId() { + return zoneId; + } + + public void setZoneId(Long zoneId) { + this.zoneId = zoneId; + } + + public Long getAccountId() { + return accountId; + } + + public void setAccountId(Long accountId) { + this.accountId = accountId; + } + + public Long getDomainId() { + return domainId; + } + + public void setDomainId(Long domainId) { + this.domainId = domainId; + } + + public Long getUsageItemId() { + return usageItemId; + } + + public void setUsageItemId(Long usageItemId) { + this.usageItemId = usageItemId; + } + + public int getUsageType() { + return usageType; + } + + public void setUsageType(int usageType) { + this.usageType = usageType; + } + + public BigDecimal getQuotaUsed() { + return quotaUsed; + } + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public Long getResourceId() { + return resourceId; + } + + public void setResourceId(Long resourceId) { + this.resourceId = resourceId; + } + + public Long getNetworkId() { + return networkId; + } + + public void setNetworkId(Long networkId) { + this.networkId = networkId; + } + + public Long getOfferingId() { + return offeringId; + } + + public void setOfferingId(Long offeringId) { + this.offeringId = offeringId; + } + + public QuotaUsageJoinVO () { + } + + @Override + public String toString() { + return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "zoneId", "accountId", "domainId", "usageItemId", "usageType", "quotaUsed", "startDate", + "endDate", "resourceId"); + } +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java new file mode 100644 index 00000000000..92482423100 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/vo/QuotaUsageResourceVO.java @@ -0,0 +1,62 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.quota.vo; + +import java.util.Date; + +public class QuotaUsageResourceVO { + private String uuid; + private String name; + private Date removed; + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } + + public boolean isRemoved() { + return this.removed != null; + } + + public QuotaUsageResourceVO(String uuid, String name, Date removed) { + this.uuid = uuid; + this.name = name; + this.removed = removed; + } +} diff --git a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml index 304b23b7220..5ca2679c388 100644 --- a/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml +++ b/framework/quota/src/main/resources/META-INF/cloudstack/quota/spring-framework-quota-context.xml @@ -26,7 +26,9 @@ - + + + diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index 85397503587..bcdbc3b46ce 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -31,14 +31,22 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.HostTagVO; import com.cloud.hypervisor.Hypervisor; +import com.cloud.network.Network; +import com.cloud.network.dao.NetworkVO; +import com.cloud.network.vpc.VpcOfferingVO; +import com.cloud.network.vpc.VpcVO; +import com.cloud.network.vpc.dao.VpcOfferingDao; import com.cloud.storage.StoragePoolTagVO; +import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.acl.RoleType; import org.apache.cloudstack.acl.RoleVO; import org.apache.cloudstack.acl.dao.RoleDao; import org.apache.cloudstack.backup.BackupOfferingVO; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.quota.constant.QuotaTypes; +import org.apache.cloudstack.quota.dao.NetworkDao; import org.apache.cloudstack.quota.dao.VmTemplateDao; +import org.apache.cloudstack.quota.dao.VpcDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -188,6 +196,15 @@ public class PresetVariableHelperTest { @Mock BackupOfferingDao backupOfferingDaoMock; + @Mock + NetworkDao networkDaoMock; + + @Mock + VpcDao vpcDaoMock; + + @Mock + VpcOfferingDao vpcOfferingDaoMock; + List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); List templateAndIsoUsageTypes = Arrays.asList(UsageTypes.TEMPLATE, UsageTypes.ISO); @@ -223,6 +240,10 @@ public class PresetVariableHelperTest { value.setVmSnapshotType(VMSnapshot.Type.Disk.toString()); value.setComputingResources(getComputingResourcesForTests()); value.setVolumeType(Volume.Type.DATADISK.toString()); + value.setState(Network.State.Implemented.toString()); + value.setResourceCounting(getResourceCountingForTests()); + value.setNetworkOffering(getNetworkOfferingForTests()); + value.setVpcOffering(getVpcOfferingForTests()); return value; } @@ -259,6 +280,13 @@ public class PresetVariableHelperTest { return configuration; } + private ResourceCounting getResourceCountingForTests() { + ResourceCounting resourceCounting = new ResourceCounting(); + resourceCounting.setRunningVms(1); + resourceCounting.setStoppedVms(1); + return resourceCounting; + } + private List getHostTagsForTests() { return Arrays.asList(new HostTagVO(1, "tag1", false), new HostTagVO(1, "tag2", false)); } @@ -324,6 +352,20 @@ public class PresetVariableHelperTest { return diskOffering; } + private GenericPresetVariable getNetworkOfferingForTests() { + GenericPresetVariable networkOffering = new GenericPresetVariable(); + networkOffering.setId("network_offering_id"); + networkOffering.setName("network_offering_name"); + return networkOffering; + } + + private GenericPresetVariable getVpcOfferingForTests() { + GenericPresetVariable vpcOffering = new GenericPresetVariable(); + vpcOffering.setId("vpc_offering_id"); + vpcOffering.setName("vpc_offering_name"); + return vpcOffering; + } + private void mockMethodValidateIfObjectIsNull() { Mockito.doNothing().when(presetVariableHelperSpy).validateIfObjectIsNull(Mockito.any(), Mockito.anyLong(), Mockito.anyString()); } @@ -1289,4 +1331,120 @@ public class PresetVariableHelperTest { Mockito.when(imageStoreDaoMock.findById(1L)).thenReturn(store); Assert.assertNotNull(presetVariableHelperSpy.getSnapshotImageStoreRef(1L, 1L)); } + + @Test + public void loadPresetVariableValueForNetworkTestRecordIsNotANetworkDoNothing() { + getQuotaTypesForTests(UsageTypes.NETWORK).forEach(type -> { + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.loadPresetVariableValueForNetwork(usageVoMock, null); + }); + + Mockito.verifyNoInteractions(networkDaoMock); + } + + @Test + public void loadPresetVariableValueForNetworkTestRecordIsNetworkSetFields() { + Value expected = getValueForTests(); + + NetworkVO networkVoMock = Mockito.mock(NetworkVO.class); + Mockito.doReturn(networkVoMock).when(networkDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + mockMethodValidateIfObjectIsNull(); + + Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); + Mockito.doReturn(expected.getState()).when(usageVoMock).getState(); + Mockito.doReturn(expected.getResourceCounting()).when(presetVariableHelperSpy).getPresetVariableValueNetworkResourceCounting(Mockito.anyLong()); + Mockito.doReturn(expected.getNetworkOffering()).when(presetVariableHelperSpy).getPresetVariableValueNetworkOffering(Mockito.anyLong()); + Mockito.doReturn(UsageTypes.NETWORK).when(usageVoMock).getUsageType(); + + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForNetwork(usageVoMock, result); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getState(), result.getState()); + Assert.assertEquals(expected.getResourceCounting(), result.getResourceCounting()); + } + + @Test + public void getPresetVariableValueNetworkResourceCountingTestSetValueAndReturnObject() { + VMInstanceVO vmInstanceVoMock1 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock1.setState(VirtualMachine.State.Stopped); + + VMInstanceVO vmInstanceVoMock2 = Mockito.spy(VMInstanceVO.class); + vmInstanceVoMock2.setState(VirtualMachine.State.Running); + + Mockito.doReturn(List.of(vmInstanceVoMock1, vmInstanceVoMock2)).when(vmInstanceDaoMock).listNonRemovedVmsByTypeAndNetwork(Mockito.anyLong(), Mockito.any()); + + mockMethodValidateIfObjectIsNull(); + + ResourceCounting expected = getResourceCountingForTests(); + + ResourceCounting result = presetVariableHelperSpy.getPresetVariableValueNetworkResourceCounting(1L); + + Assert.assertEquals(expected.getRunningVms(), result.getRunningVms()); + Assert.assertEquals(expected.getStoppedVms(), result.getStoppedVms()); + } + + @Test + public void loadPresetVariableValueForVpcTestRecordIsNotAVpcDoNothing() { + getQuotaTypesForTests(UsageTypes.VPC).forEach(type -> { + Mockito.doReturn(type.getKey()).when(usageVoMock).getUsageType(); + presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, null); + }); + + Mockito.verifyNoInteractions(vpcDaoMock); + } + + @Test + public void loadPresetVariableValueForVpcTestRecordIsVpcSetFields() { + Value expected = getValueForTests(); + + VpcVO networkVoMock = Mockito.mock(VpcVO.class); + Mockito.doReturn(networkVoMock).when(vpcDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + mockMethodValidateIfObjectIsNull(); + + Mockito.doReturn(expected.getId()).when(networkVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkVoMock).getName(); + Mockito.doReturn(expected.getVpcOffering()).when(presetVariableHelperSpy).getPresetVariableValueVpcOffering(Mockito.anyLong()); + + Mockito.doReturn(UsageTypes.VPC).when(usageVoMock).getUsageType(); + + Value result = new Value(); + presetVariableHelperSpy.loadPresetVariableValueForVpc(usageVoMock, result); + + assertPresetVariableIdAndName(expected, result); + Assert.assertEquals(expected.getVpcOffering(), result.getVpcOffering()); + } + + @Test + public void getPresetVariableValueNetworkOfferingTestSetValuesAndReturnObject() { + NetworkOfferingVO networkOfferingVoMock = Mockito.mock(NetworkOfferingVO.class); + Mockito.doReturn(networkOfferingVoMock).when(networkOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + + GenericPresetVariable expected = getGenericPresetVariableForTests(); + Mockito.doReturn(expected.getId()).when(networkOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(networkOfferingVoMock).getName(); + + GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueNetworkOffering(1L); + + assertPresetVariableIdAndName(expected, result); + } + + @Test + public void getPresetVariableValueVpcOfferingTestSetValuesAndReturnObject() { + VpcOfferingVO vpcOfferingVoMock = Mockito.mock(VpcOfferingVO.class); + Mockito.doReturn(vpcOfferingVoMock).when(vpcOfferingDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + mockMethodValidateIfObjectIsNull(); + + GenericPresetVariable expected = getGenericPresetVariableForTests(); + Mockito.doReturn(expected.getId()).when(vpcOfferingVoMock).getUuid(); + Mockito.doReturn(expected.getName()).when(vpcOfferingVoMock).getName(); + + GenericPresetVariable result = presetVariableHelperSpy.getPresetVariableValueVpcOffering(1L); + + assertPresetVariableIdAndName(expected, result); + } } diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java new file mode 100644 index 00000000000..afdd88f8d1d --- /dev/null +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/dao/QuotaBalanceDaoImplTest.java @@ -0,0 +1,91 @@ + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.Arrays; + +import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class QuotaBalanceDaoImplTest { + QuotaBalanceDaoImpl quotaBalanceDaoImplSpy = Mockito.spy(QuotaBalanceDaoImpl.class); + + @Mock + QuotaBalanceVO quotaBalanceVoMock; + + @Test + public void getLastQuotaBalanceTestLastEntryIsNullAndNoCreditsReturnsZero() { + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(1L, 2L); + + Assert.assertEquals(BigDecimal.ZERO, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryAndNoCredits() { + BigDecimal expected = BigDecimal.valueOf(-1542.46); + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(new ArrayList<>()).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryPlusCredits() { + BigDecimal balance = BigDecimal.valueOf(-1542.46); + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = balance.add(credit1).add(credit2); + + Mockito.doReturn(quotaBalanceVoMock).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(balance, credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } + + @Test + public void getLastQuotaBalanceTestReturnsLastEntryIsNullPlusCredits() { + BigDecimal credit1 = new BigDecimal("150.14"); + BigDecimal credit2 = new BigDecimal("78.96"); + BigDecimal expected = credit1.add(credit2); + + Mockito.doReturn(null).when(quotaBalanceDaoImplSpy).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(credit1, credit2).when(quotaBalanceVoMock).getCreditBalance(); + Mockito.doReturn(Arrays.asList(quotaBalanceVoMock, quotaBalanceVoMock)).when(quotaBalanceDaoImplSpy).findCreditBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + + BigDecimal result = quotaBalanceDaoImplSpy.getLastQuotaBalance(5L, 8L); + + Assert.assertEquals(expected, result); + } +} diff --git a/packaging/el8/cloud.spec b/packaging/el8/cloud.spec index 705959336f1..13c0a36cb10 100644 --- a/packaging/el8/cloud.spec +++ b/packaging/el8/cloud.spec @@ -83,6 +83,8 @@ Requires: (iptables-services or iptables) Requires: rng-tools Requires: (qemu-img or qemu-tools) Requires: python3-pip +Requires: python3-six +Requires: python3-protobuf Requires: python3-setuptools Requires: (libgcrypt > 1.8.3 or libgcrypt20) Group: System Environment/Libraries @@ -115,7 +117,7 @@ Requires: ipset Requires: perl Requires: rsync Requires: cifs-utils -Requires: edk2-ovmf +Requires: (edk2-ovmf or qemu-ovmf-x86_64) Requires: swtpm Requires: (python3-libvirt or python3-libvirt-python) Requires: (qemu-img or qemu-tools) @@ -334,11 +336,11 @@ cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json ln -sf /etc/%{name}/ui/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json -# Package mysql-connector-python -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/ee/ff/48bde5c0f013094d729fe4b0316ba2a24774b3ff1c52d924a8a4cb04078a/six-1.15.0-py2.py3-none-any.whl -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/e9/93/4860cebd5ad3ff2664ad3c966490ccb46e3b88458b2095145bca11727ca4/setuptools-47.3.1-py3-none-any.whl -wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/32/27/1141a8232723dcb10a595cc0ce4321dcbbd5215300bf4acfc142343205bf/protobuf-3.19.6-py2.py3-none-any.whl +# Package mysql-connector-python (bundled to avoid dependency on external community repo) +# Version 8.0.31 is the last version supporting Python 3.6 (EL8) wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Version 8.3.0 supports Python 3.8 to 3.12 (EL9, EL10) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/53/ed/26a4b8cacb8852c6fd97d2d58a7f2591c41989807ea82bd8d9725a4e6937/mysql_connector_python-8.3.0-py2.py3-none-any.whl chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt @@ -455,8 +457,13 @@ then fi %post management -# Install mysql-connector-python -pip3 install %{_datadir}/%{name}-management/setup/wheel/six-1.15.0-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/setuptools-47.3.1-py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/protobuf-3.19.6-py2.py3-none-any.whl %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Install mysql-connector-python wheel +# Detect Python version to install compatible wheel +if python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 7) else 1)'; then + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.3.0-py2.py3-none-any.whl +else + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +fi /usr/bin/systemctl enable cloudstack-management > /dev/null 2>&1 || true /usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true diff --git a/packaging/suse15 b/packaging/suse15 deleted file mode 120000 index 4dad90d45e0..00000000000 --- a/packaging/suse15 +++ /dev/null @@ -1 +0,0 @@ -el8 \ No newline at end of file diff --git a/packaging/suse15/cloud-ipallocator.rc b/packaging/suse15/cloud-ipallocator.rc new file mode 120000 index 00000000000..647598e6dc4 --- /dev/null +++ b/packaging/suse15/cloud-ipallocator.rc @@ -0,0 +1 @@ +../el8/cloud-ipallocator.rc \ No newline at end of file diff --git a/packaging/suse15/cloud.limits b/packaging/suse15/cloud.limits new file mode 120000 index 00000000000..37be77e3acf --- /dev/null +++ b/packaging/suse15/cloud.limits @@ -0,0 +1 @@ +../el8/cloud.limits \ No newline at end of file diff --git a/packaging/suse15/cloud.spec b/packaging/suse15/cloud.spec new file mode 100644 index 00000000000..cdfc5a72a34 --- /dev/null +++ b/packaging/suse15/cloud.spec @@ -0,0 +1,750 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +%define __os_install_post %{nil} +%global debug_package %{nil} +%global __requires_exclude libc\\.so\\..*|libc\\.so\\.6\\(GLIBC_.*\\) +%define _binaries_in_noarch_packages_terminate_build 0 + +# DISABLE the post-percentinstall java repacking and line number stripping +# we need to find a way to just disable the java repacking and line number stripping, but not the autodeps + +Name: cloudstack +Summary: CloudStack IaaS Platform +#http://fedoraproject.org/wiki/PackageNamingGuidelines#Pre-Release_packages +%define _maventag %{_fullver} +Release: %{_rel} + +Version: %{_ver} +License: ASL 2.0 +Vendor: Apache CloudStack +Packager: Apache CloudStack +Group: System Environment/Libraries +# FIXME do groups for every single one of the subpackages +Source0: %{name}-%{_maventag}.tgz +BuildRoot: %{_tmppath}/%{name}-%{_maventag}-%{release}-build +BuildArch: noarch + +BuildRequires: (java-11-openjdk-devel or java-17-openjdk-devel or java-21-openjdk-devel) +#BuildRequires: ws-commons-util +BuildRequires: jpackage-utils +BuildRequires: gcc +BuildRequires: glibc-devel +BuildRequires: /usr/bin/mkisofs +BuildRequires: python3-setuptools +BuildRequires: wget +BuildRequires: nodejs + +%description +CloudStack is a highly-scalable elastic, open source, +intelligent IaaS cloud implementation. + +%package management +Summary: CloudStack management server UI +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Requires: python3 +Requires: bash +Requires: gawk +Requires: which +Requires: file +Requires: tar +Requires: bzip2 +Requires: gzip +Requires: unzip +Requires: (/sbin/mount.nfs or /usr/sbin/mount.nfs) +Requires: (openssh-clients or openssh) +Requires: (nfs-utils or nfs-client) +Requires: iproute +Requires: wget +Requires: (mysql or mariadb or mysql8.4) +Requires: sudo +Requires: /sbin/service +Requires: /sbin/chkconfig +Requires: /usr/bin/ssh-keygen +Requires: (genisoimage or mkisofs or xorrisofs) +Requires: ipmitool +Requires: %{name}-common = %{_ver} +Requires: (iptables-services or iptables) +Requires: rng-tools +Requires: (qemu-img or qemu-tools) +Requires: python3-pip +Requires: python3-six +Requires: python3-protobuf +Requires: python3-setuptools +Requires: (libgcrypt > 1.8.3 or libgcrypt20) +Group: System Environment/Libraries +%description management +The CloudStack management server is the central point of coordination, +management, and intelligence in CloudStack. + +%package common +Summary: Apache CloudStack common files and scripts +Requires: python3 +Group: System Environment/Libraries +%description common +The Apache CloudStack files shared between agent and management server +%global __requires_exclude libc\\.so\\..*|libc\\.so\\.6\\(GLIBC_.*\\)|^(libuuid\\.so\\.1|/usr/bin/python)$ + +%package agent +Summary: CloudStack Agent for KVM hypervisors +Requires: (openssh-clients or openssh) +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Requires: %{name}-common = %{_ver} +Requires: libvirt +Requires: libvirt-daemon-driver-storage-rbd +Requires: ebtables +Requires: iptables +Requires: ethtool +Requires: (net-tools or net-tools-deprecated) +Requires: iproute +Requires: ipset +Requires: perl +Requires: rsync +Requires: cifs-utils +Requires: (edk2-ovmf or qemu-ovmf-x86_64) +Requires: swtpm +Requires: (python3-libvirt or python3-libvirt-python) +Requires: (qemu-img or qemu-tools) +Requires: qemu-kvm +Requires: cryptsetup +Requires: rng-tools +Requires: (libgcrypt > 1.8.3 or libgcrypt20) +Requires: (selinux-tools if selinux-tools) +Requires: sysstat +Provides: cloud-agent +Group: System Environment/Libraries +%description agent +The CloudStack agent for KVM hypervisors + +%package baremetal-agent +Summary: CloudStack baremetal agent +Requires: tftp-server +Requires: xinetd +Requires: syslinux +Requires: chkconfig +Requires: dhcp +Requires: httpd +Group: System Environment/Libraries +%description baremetal-agent +The CloudStack baremetal agent + +%package usage +Summary: CloudStack Usage calculation server +Requires: (java-17-openjdk or java-21-openjdk) +Requires: (tzdata-java or timezone-java) +Group: System Environment/Libraries +%description usage +The CloudStack usage calculation service + +%package ui +Summary: CloudStack UI +Group: System Environment/Libraries +%description ui +The CloudStack UI + +%package marvin +Summary: Apache CloudStack Marvin library +Requires: python3-pip +Requires: gcc +Requires: python3-devel +Requires: libffi-devel +Requires: openssl-devel +Group: System Environment/Libraries +%description marvin +Apache CloudStack Marvin library + +%package integration-tests +Summary: Apache CloudStack Marvin integration tests +Requires: %{name}-marvin = %{_ver} +Group: System Environment/Libraries +%description integration-tests +Apache CloudStack Marvin integration tests + +%if "%{_ossnoss}" == "noredist" +%package mysql-ha +Summary: Apache CloudStack Balancing Strategy for MySQL +Group: System Environmnet/Libraries +%description mysql-ha +Apache CloudStack Balancing Strategy for MySQL + +%endif + +%prep +echo Doing CloudStack build + +%setup -q -n %{name}-%{_maventag} + +%build + +cp packaging/suse15/replace.properties build/replace.properties +echo VERSION=%{_maventag} >> build/replace.properties +echo PACKAGE=%{name} >> build/replace.properties +touch build/gitrev.txt +echo $(git rev-parse HEAD) > build/gitrev.txt + +if [ "%{_ossnoss}" == "NOREDIST" -o "%{_ossnoss}" == "noredist" ] ; then + echo "Adding noredist flag to the maven build" + FLAGS="$FLAGS -Dnoredist" +fi + +if [ "%{_sim}" == "SIMULATOR" -o "%{_sim}" == "simulator" ] ; then + echo "Adding simulator flag to the maven build" + FLAGS="$FLAGS -Dsimulator" +fi + +if [ \"%{_temp}\" != "" ]; then + echo "Adding flags to package requested templates" + FLAGS="$FLAGS `rpm --eval %{?_temp}`" +fi + +mvn -Psystemvm,developer $FLAGS clean package +cd ui && npm install && npm run build && cd .. + +%install +[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT} +# Common directories +mkdir -p ${RPM_BUILD_ROOT}%{_bindir} +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/ipallocator +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_initrddir} +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/default +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d + +# Common +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site +mkdir -p ${RPM_BUILD_ROOT}/usr/bin +cp -r scripts/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +install -D systemvm/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/vms/ +install python/lib/cloud_utils.py ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloud_utils.py +cp -r python/lib/cloudutils ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/ +python3 -m py_compile ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloud_utils.py +python3 -m compileall ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/python-site/cloudutils +cp build/gitrev.txt ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts +cp packaging/suse15/cloudstack-sccs ${RPM_BUILD_ROOT}/usr/bin + +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco +cp -r plugins/network-elements/cisco-vnmc/src/main/scripts/network/cisco/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/scripts/network/cisco + +# Management +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/run +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel + +# Setup Jetty +ln -sf /etc/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/conf +ln -sf /var/log/%{name}/management ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/logs + +install -D client/target/utilities/bin/cloud-migrate-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-migrate-databases +install -D client/target/utilities/bin/cloud-set-guest-password ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-password +install -D client/target/utilities/bin/cloud-set-guest-sshkey ${RPM_BUILD_ROOT}%{_bindir}/%{name}-set-guest-sshkey +install -D client/target/utilities/bin/cloud-setup-databases ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-databases +install -D client/target/utilities/bin/cloud-setup-encryption ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-encryption +install -D client/target/utilities/bin/cloud-setup-management ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-management +install -D client/target/utilities/bin/cloud-setup-baremetal ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-baremetal +install -D client/target/utilities/bin/cloud-sysvmadm ${RPM_BUILD_ROOT}%{_bindir}/%{name}-sysvmadm +install -D client/target/utilities/bin/cloud-update-xenserver-licenses ${RPM_BUILD_ROOT}%{_bindir}/%{name}-update-xenserver-licenses +# Bundle cmk in cloudstack-management +wget https://github.com/apache/cloudstack-cloudmonkey/releases/latest/download/cmk.linux.x86-64 -O ${RPM_BUILD_ROOT}%{_bindir}/cmk +chmod +x ${RPM_BUILD_ROOT}%{_bindir}/cmk + +cp -r client/target/utilities/scripts/db/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup +cp -r plugins/integrations/kubernetes-service/src/main/resources/conf/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf +cp -r client/target/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/ +cp -r client/target/classes/META-INF/webapp ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp +cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/ +cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/ +rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/config.json +ln -sf /etc/%{name}/management/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapp/config.json +mv ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cloud-client-ui-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/cloudstack-%{_maventag}.jar +cp client/target/lib/*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/lib/ + +# Don't package the scripts in the management webapp +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/scripts +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/vms + +for name in db.properties server.properties log4j-cloud.xml environment.properties java.security.ciphers +do + cp client/target/conf/$name ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/$name +done + +ln -sf log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/management/log4j2.xml + +install python/bindir/cloud-external-ipallocator.py ${RPM_BUILD_ROOT}%{_bindir}/%{name}-external-ipallocator.py +install -D client/target/pythonlibs/jasypt-1.9.3.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +install -D utils/target/cloud-utils-%{_maventag}-bundled.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-common/lib/%{name}-utils.jar + +install -D packaging/suse15/cloud-ipallocator.rc ${RPM_BUILD_ROOT}%{_initrddir}/%{name}-ipallocator +install -D packaging/suse15/cloud.limits ${RPM_BUILD_ROOT}%{_sysconfdir}/security/limits.d/cloud +install -D packaging/suse15/filelimit.conf ${RPM_BUILD_ROOT}%{_sysconfdir}/systemd/system/%{name}-management.service.d +install -D packaging/systemd/cloudstack-management.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-management.service +install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-management +install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management +touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid +#install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina +install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management + +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/etcd-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/etcd-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-control-node-add.yml +install -D plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/cks/conf/k8s-node.yml + +# SystemVM template +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm +cp -r engine/schema/dist/systemvm-templates/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm +rm -rf ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm/sha512sum.txt + +# Sample Extensions +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/extensions +cp -r extensions/* ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/extensions +ln -sf %{_sysconfdir}/%{name}/extensions ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/extensions + +# UI +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/ui +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ +cp -r client/target/classes/META-INF/webapp/WEB-INF ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui +cp ui/dist/config.json ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/ui/ +cp -r ui/dist/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/ +rm -f ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json +ln -sf /etc/%{name}/ui/config.json ${RPM_BUILD_ROOT}%{_datadir}/%{name}-ui/config.json + +# Package mysql-connector-python (bundled to avoid dependency on external community repo) +# Version 8.0.31 is the last version supporting Python 3.6 (EL8) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +# Version 8.3.0 supports Python 3.8 to 3.12 (EL9, EL10) +wget -P ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/setup/wheel https://files.pythonhosted.org/packages/53/ed/26a4b8cacb8852c6fd97d2d58a7f2591c41989807ea82bd8d9725a4e6937/mysql_connector_python-8.3.0-py2.py3-none-any.whl + +chmod 440 ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/mnt +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/%{name}/management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/work +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/cache/%{name}/management/temp +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/management +chmod 770 ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent + +# KVM Agent +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/agent +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/plugins +install -D packaging/systemd/cloudstack-agent.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-agent.service +install -D packaging/systemd/cloudstack-rolling-maintenance@.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-rolling-maintenance@.service +install -D packaging/systemd/cloudstack-agent.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-agent +install -D agent/target/transformed/agent.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/agent.properties +install -D agent/target/transformed/uefi.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/uefi.properties +install -D agent/target/transformed/environment.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/environment.properties +install -D agent/target/transformed/log4j-cloud.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/agent/log4j-cloud.xml +install -D agent/target/transformed/cloud-setup-agent ${RPM_BUILD_ROOT}%{_bindir}/%{name}-setup-agent +install -D agent/target/transformed/cloudstack-agent-upgrade ${RPM_BUILD_ROOT}%{_bindir}/%{name}-agent-upgrade +install -D agent/target/transformed/cloud-guest-tool ${RPM_BUILD_ROOT}%{_bindir}/%{name}-guest-tool +install -D agent/target/transformed/libvirtqemuhook ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook +install -D agent/target/transformed/rolling-maintenance ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/rolling-maintenance +install -D agent/target/transformed/cloud-ssh ${RPM_BUILD_ROOT}%{_bindir}/%{name}-ssh +install -D agent/target/transformed/cloudstack-agent-profile.sh ${RPM_BUILD_ROOT}%{_sysconfdir}/profile.d/%{name}-agent-profile.sh +install -D agent/target/transformed/cloudstack-agent.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-agent +install -D plugins/hypervisors/kvm/target/cloud-plugin-hypervisor-kvm-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%name-agent/lib/cloud-plugin-hypervisor-kvm-%{_maventag}.jar +cp plugins/hypervisors/kvm/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +cp plugins/storage/volume/storpool/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib +cp plugins/storage/volume/linstor/target/*.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib + +# Usage server +mkdir -p ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib +install -D usage/target/cloud-usage-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/cloud-usage-%{_maventag}.jar +install -D usage/target/transformed/db.properties ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/db.properties +install -D usage/target/transformed/log4j-cloud_usage.xml ${RPM_BUILD_ROOT}%{_sysconfdir}/%{name}/usage/log4j-cloud.xml +cp usage/target/dependencies/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ +cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/ +install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service +install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage +mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/ +install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage + +# Marvin +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin +cp tools/marvin/dist/Marvin-*.tar.gz ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin/ + +# integration-tests +mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests +cp -r test/integration/* ${RPM_BUILD_ROOT}%{_datadir}/%{name}-integration-tests/ + +# MYSQL HA +if [ "x%{_ossnoss}" == "xnoredist" ] ; then + mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib + cp -r plugins/database/mysql-ha/target/cloud-plugin-database-mysqlha-%{_maventag}.jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-mysql-ha/lib +fi + +#License files from whisker +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-management-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-common-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-ui-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-ui-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE +install -D tools/whisker/NOTICE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE +install -D tools/whisker/LICENSE ${RPM_BUILD_ROOT}%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE + +%clean +[ ${RPM_BUILD_ROOT} != "/" ] && rm -rf ${RPM_BUILD_ROOT} + +%posttrans common + +unalias cp +python_dir=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +if [ ! -z $python_dir ];then + cp -f -r /usr/share/cloudstack-common/python-site/* $python_dir/ +fi + +%preun management +/usr/bin/systemctl stop cloudstack-management || true +/usr/bin/systemctl disable cloudstack-management || true + +%pre management +id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -U -c "CloudStack unprivileged user" \ + -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud || true + +rm -rf %{_localstatedir}/cache/cloudstack + +# in case of upgrade to 4.9+ copy commands.properties if not exists in /etc/cloudstack/management/ +if [ "$1" == "2" ] ; then + if [ -f "%{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties" ] && [ ! -f "%{_sysconfdir}/%{name}/management/commands.properties" ] ; then + cp -p %{_datadir}/%{name}-management/webapps/client/WEB-INF/classes/commands.properties %{_sysconfdir}/%{name}/management/commands.properties + fi +fi + +# Remove old tomcat symlinks and env config file +if [ -L "%{_datadir}/%{name}-management/lib" ] +then + rm -f %{_datadir}/%{name}-management/bin + rm -f %{_datadir}/%{name}-management/lib + rm -f %{_datadir}/%{name}-management/temp + rm -f %{_datadir}/%{name}-management/work + rm -f %{_sysconfdir}/default/%{name}-management +fi + +%post management +# Install mysql-connector-python wheel +# Detect Python version to install compatible wheel +if python3 -c 'import sys; sys.exit(0 if sys.version_info >= (3, 7) else 1)'; then + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.3.0-py2.py3-none-any.whl +else + pip3 install %{_datadir}/%{name}-management/setup/wheel/mysql_connector_python-8.0.31-py2.py3-none-any.whl +fi + +/usr/bin/systemctl enable cloudstack-management > /dev/null 2>&1 || true +/usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true + +grep -s -q "db.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.cloud.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "db.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.usage.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "db.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" || sed -i -e "\$adb.simulator.driver=jdbc:mysql" "%{_sysconfdir}/%{name}/management/db.properties" + +# Update DB properties having master and slave(s), with source and replica(s) respectively (for inclusiveness) +grep -s -q "^db.cloud.slaves=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.slaves=/db.cloud.replicas=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.cloud.secondsBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.secondsBeforeRetryMaster=/db.cloud.secondsBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.cloud.queriesBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.cloud.queriesBeforeRetryMaster=/db.cloud.queriesBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.slaves=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.slaves=/db.usage.replicas=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.secondsBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.secondsBeforeRetryMaster=/db.usage.secondsBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" +grep -s -q "^db.usage.queriesBeforeRetryMaster=" "%{_sysconfdir}/%{name}/management/db.properties" && sed -i "s/^db.usage.queriesBeforeRetryMaster=/db.usage.queriesBeforeRetrySource=/g" "%{_sysconfdir}/%{name}/management/db.properties" + +if [ ! -f %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/vhd-util ] ; then + echo Please download vhd-util from http://download.cloudstack.org/tools/vhd-util and put it in + echo %{_datadir}/cloudstack-common/scripts/vm/hypervisor/xenserver/ +fi + +if [ -f %{_sysconfdir}/sysconfig/%{name}-management ] ; then + rm -f %{_sysconfdir}/sysconfig/%{name}-management +fi + +chown -R cloud:cloud /var/log/cloudstack/management +chown -R cloud:cloud /usr/share/cloudstack-management/templates +find /usr/share/cloudstack-management/templates -type d -exec chmod 0770 {} \; + +systemctl daemon-reload + +%posttrans management +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text management +fi + +%preun agent +/sbin/service cloudstack-agent stop || true +if [ "$1" == "0" ] ; then + /sbin/chkconfig --del cloudstack-agent > /dev/null 2>&1 || true +fi + +%pre agent + +# save old configs if they exist (for upgrade). Otherwise we may lose them +# when the old packages are erased. There are a lot of properties files here. +if [ -d "%{_sysconfdir}/cloud" ] ; then + mv %{_sysconfdir}/cloud %{_sysconfdir}/cloud.rpmsave +fi + +%posttrans agent + +if [ "$1" == "2" ] ; then + echo "Running %{_bindir}/%{name}-agent-upgrade to update bridge name for upgrade from CloudStack 4.0.x (and before) to CloudStack 4.1 (and later)" + %{_bindir}/%{name}-agent-upgrade +fi +if [ ! -d %{_sysconfdir}/libvirt/hooks ] ; then + mkdir %{_sysconfdir}/libvirt/hooks +fi +cp -a ${RPM_BUILD_ROOT}%{_datadir}/%{name}-agent/lib/libvirtqemuhook %{_sysconfdir}/libvirt/hooks/qemu +mkdir -m 0755 -p /usr/share/cloudstack-agent/tmp +/usr/bin/systemctl restart libvirtd +/usr/bin/systemctl enable cloudstack-agent > /dev/null 2>&1 || true +/usr/bin/systemctl enable cloudstack-rolling-maintenance@p > /dev/null 2>&1 || true +/usr/bin/systemctl enable --now rngd > /dev/null 2>&1 || true + +# if saved agent.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/agent.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/agent.properties %{_sysconfdir}/%{name}/agent/agent.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/agent.properties %{_sysconfdir}/cloud.rpmsave/agent/agent.properties.rpmsave +fi + +# if saved uefi.properties from upgrade exist, copy them over +if [ -f "%{_sysconfdir}/cloud.rpmsave/agent/uefi.properties" ]; then + mv %{_sysconfdir}/%{name}/agent/uefi.properties %{_sysconfdir}/%{name}/agent/uefi.properties.rpmnew + cp -p %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/%{name}/agent + # make sure we only do this on the first install of this RPM, don't want to overwrite on a reinstall + mv %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties %{_sysconfdir}/cloud.rpmsave/agent/uefi.properties.rpmsave +fi + +systemctl daemon-reload + +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text agent +fi + +%pre usage +id cloud > /dev/null 2>&1 || /usr/sbin/useradd -M -U -c "CloudStack unprivileged user" \ + -r -s /bin/sh -d %{_localstatedir}/cloudstack/management cloud|| true + +%preun usage +/sbin/service cloudstack-usage stop || true +if [ "$1" == "0" ] ; then + /sbin/chkconfig --del cloudstack-usage > /dev/null 2>&1 || true +fi + +%post usage +if [ -f "%{_sysconfdir}/%{name}/management/db.properties" ]; then + echo "Replacing usage server's db.properties with a link to the management server's db.properties" + rm -f %{_sysconfdir}/%{name}/usage/db.properties + ln -s %{_sysconfdir}/%{name}/management/db.properties %{_sysconfdir}/%{name}/usage/db.properties + /usr/bin/systemctl enable cloudstack-usage > /dev/null 2>&1 || true +fi + +if [ -f "%{_sysconfdir}/%{name}/management/key" ]; then + echo "Replacing usage server's key with a link to the management server's key" + rm -f %{_sysconfdir}/%{name}/usage/key + ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key +fi + +if [ ! -f "%{_sysconfdir}/%{name}/usage/key" ]; then + ln -s %{_sysconfdir}/%{name}/management/key %{_sysconfdir}/%{name}/usage/key +fi + +mkdir -p /usr/local/libexec +if [ ! -f "/usr/local/libexec/sanity-check-last-id" ]; then + echo 1 > /usr/local/libexec/sanity-check-last-id +fi +chown cloud:cloud /usr/local/libexec/sanity-check-last-id + +%posttrans usage +# Print help message +if [ -f "/usr/share/cloudstack-common/scripts/installer/cloudstack-help-text" ];then + sed -i "s,^ACS_VERSION=.*,ACS_VERSION=%{_maventag},g" /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text + /usr/share/cloudstack-common/scripts/installer/cloudstack-help-text usage +fi + +%post marvin +pip3 install --upgrade https://files.pythonhosted.org/packages/08/1f/42d74bae9dd6dcfec67c9ed0f3fa482b1ae5ac5f117ca82ab589ecb3ca19/mysql_connector_python-8.0.31-py2.py3-none-any.whl +pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz + +#No default permission as the permission setup is complex +%files management +%defattr(-,root,root,-) +%dir %{_datadir}/%{name}-management +%dir %attr(0770,root,cloud) %{_localstatedir}/%{name}/mnt +%dir %attr(0770,cloud,cloud) %{_localstatedir}/%{name}/management +%dir %attr(0770,root,cloud) %{_localstatedir}/cache/%{name}/management +%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/management +%config(noreplace) %{_sysconfdir}/default/%{name}-management +%config(noreplace) %{_sysconfdir}/sudoers.d/%{name}-management +%config(noreplace) %{_sysconfdir}/security/limits.d/cloud +%config(noreplace) %{_sysconfdir}/systemd/system/%{name}-management.service.d +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/db.properties +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/server.properties +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/management/config.json +%config(noreplace) %{_sysconfdir}/%{name}/management/log4j-cloud.xml +%config(noreplace) %{_sysconfdir}/%{name}/management/log4j2.xml +%config(noreplace) %{_sysconfdir}/%{name}/management/environment.properties +%config(noreplace) %{_sysconfdir}/%{name}/management/java.security.ciphers +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management +%attr(0644,root,root) %{_unitdir}/%{name}-management.service +%attr(0755,cloud,cloud) %{_localstatedir}/run/%{name}-management.pid +%attr(0755,root,root) %{_bindir}/%{name}-setup-management +%attr(0755,root,root) %{_bindir}/%{name}-update-xenserver-licenses +%{_datadir}/%{name}-management/conf +%{_datadir}/%{name}-management/lib/*.jar +%{_datadir}/%{name}-management/logs +%{_datadir}/%{name}-management/templates +%{_datadir}/%{name}-management/extensions +%attr(0755,root,root) %{_bindir}/%{name}-setup-databases +%attr(0755,root,root) %{_bindir}/%{name}-migrate-databases +%attr(0755,root,root) %{_bindir}/%{name}-set-guest-password +%attr(0755,root,root) %{_bindir}/%{name}-set-guest-sshkey +%attr(0755,root,root) %{_bindir}/%{name}-sysvmadm +%attr(0755,root,root) %{_bindir}/%{name}-setup-encryption +%attr(0755,root,root) %{_bindir}/cmk +%{_datadir}/%{name}-management/cks/conf/*.yml +%{_datadir}/%{name}-management/setup/*.sql +%{_datadir}/%{name}-management/setup/*.sh +%{_datadir}/%{name}-management/setup/server-setup.xml +%{_datadir}/%{name}-management/webapp/* +%dir %attr(0770, cloud, cloud) %{_datadir}/%{name}-management/templates +%dir %attr(0770, cloud, cloud) %{_datadir}/%{name}-management/templates/systemvm +%attr(0644, cloud, cloud) %{_datadir}/%{name}-management/templates/systemvm/* +%attr(0755,root,root) %{_bindir}/%{name}-external-ipallocator.py +%attr(0755,root,root) %{_initrddir}/%{name}-ipallocator +%dir %attr(0770,root,root) %{_localstatedir}/log/%{name}/ipallocator +%{_defaultdocdir}/%{name}-management-%{version}/LICENSE +%{_defaultdocdir}/%{name}-management-%{version}/NOTICE +%{_datadir}/%{name}-management/setup/wheel/*.whl +%dir %attr(0755,cloud,cloud) %{_sysconfdir}/%{name}/extensions +%attr(0755,cloud,cloud) %{_sysconfdir}/%{name}/extensions/* + +%files agent +%attr(0755,root,root) %{_bindir}/%{name}-setup-agent +%attr(0755,root,root) %{_bindir}/%{name}-agent-upgrade +%attr(0755,root,root) %{_bindir}/%{name}-guest-tool +%attr(0755,root,root) %{_bindir}/%{name}-ssh +%attr(0644,root,root) %{_unitdir}/%{name}-agent.service +%attr(0644,root,root) %{_unitdir}/%{name}-rolling-maintenance@.service +%config(noreplace) %{_sysconfdir}/default/%{name}-agent +%attr(0644,root,root) %{_sysconfdir}/profile.d/%{name}-agent-profile.sh +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-agent +%attr(0755,root,root) %{_datadir}/%{name}-common/scripts/network/cisco +%config(noreplace) %{_sysconfdir}/%{name}/agent +%dir %{_localstatedir}/log/%{name}/agent +%attr(0644,root,root) %{_datadir}/%{name}-agent/lib/*.jar +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/libvirtqemuhook +%attr(0755,root,root) %{_datadir}/%{name}-agent/lib/rolling-maintenance +%dir %{_datadir}/%{name}-agent/plugins +%{_defaultdocdir}/%{name}-agent-%{version}/LICENSE +%{_defaultdocdir}/%{name}-agent-%{version}/NOTICE + +%files common +%dir %attr(0755,root,root) %{_datadir}/%{name}-common/python-site/cloudutils +%dir %attr(0755,root,root) %{_datadir}/%{name}-common/vms +%attr(0755,root,root) %{_datadir}/%{name}-common/scripts +%attr(0755,root,root) /usr/bin/cloudstack-sccs +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/agent.zip +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/cloud-scripts.tgz +%attr(0644, root, root) %{_datadir}/%{name}-common/vms/patch-sysvms.sh +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloud_utils.py +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/__pycache__/* +%attr(0644,root,root) %{_datadir}/%{name}-common/python-site/cloudutils/* +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/jasypt-1.9.3.jar +%attr(0644, root, root) %{_datadir}/%{name}-common/lib/%{name}-utils.jar +%{_defaultdocdir}/%{name}-common-%{version}/LICENSE +%{_defaultdocdir}/%{name}-common-%{version}/NOTICE + +%files ui +%config(noreplace) %attr(0640,root,cloud) %{_sysconfdir}/%{name}/ui/config.json +%{_datadir}/%{name}-ui/* +%{_defaultdocdir}/%{name}-ui-%{version}/LICENSE +%{_defaultdocdir}/%{name}-ui-%{version}/NOTICE + +%files usage +%attr(0644,root,root) %{_unitdir}/%{name}-usage.service +%config(noreplace) %{_sysconfdir}/default/%{name}-usage +%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage +%attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar +%attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar +%dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage +%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/db.properties +%attr(0644,root,root) %{_sysconfdir}/%{name}/usage/log4j-cloud.xml +%{_defaultdocdir}/%{name}-usage-%{version}/LICENSE +%{_defaultdocdir}/%{name}-usage-%{version}/NOTICE + +%files marvin +%attr(0644,root,root) %{_datadir}/%{name}-marvin/Marvin*.tar.gz +%{_defaultdocdir}/%{name}-marvin-%{version}/LICENSE +%{_defaultdocdir}/%{name}-marvin-%{version}/NOTICE + +%files integration-tests +%attr(0755,root,root) %{_datadir}/%{name}-integration-tests/* +%{_defaultdocdir}/%{name}-integration-tests-%{version}/LICENSE +%{_defaultdocdir}/%{name}-integration-tests-%{version}/NOTICE + +%if "%{_ossnoss}" == "noredist" +%files mysql-ha +%defattr(0644,cloud,cloud,0755) +%attr(0644,root,root) %{_datadir}/%{name}-mysql-ha/lib/* +%endif + +%files baremetal-agent +%attr(0755,root,root) %{_bindir}/cloudstack-setup-baremetal + +%changelog +* Thu Dec 22 2022 Rohit Yadav 4.18.0 +- Add support for EL9 + +* Fri Oct 14 2022 Daan Hoogland 4.18.0 +- initialising sanity check pointer file + +* Tue Jun 29 2021 David Jumani 4.16.0 +- Adding SUSE 15 support + +* Thu Apr 30 2015 Rohit Yadav 4.6.0 +- Remove awsapi package + +* Wed Nov 19 2014 Hugo Trippaers 4.6.0 +- Create a specific spec for CentOS 7 + +* Fri Jul 4 2014 Hugo Trippaers 4.5.0 +- Add a package for the mysql ha module + +* Fri Oct 5 2012 Hugo Trippaers 4.1.0 +- new style spec file diff --git a/packaging/suse15/cloudstack-agent.te b/packaging/suse15/cloudstack-agent.te new file mode 120000 index 00000000000..30e123f6cba --- /dev/null +++ b/packaging/suse15/cloudstack-agent.te @@ -0,0 +1 @@ +../el8/cloudstack-agent.te \ No newline at end of file diff --git a/packaging/suse15/cloudstack-sccs b/packaging/suse15/cloudstack-sccs new file mode 120000 index 00000000000..b9e6ed9dc08 --- /dev/null +++ b/packaging/suse15/cloudstack-sccs @@ -0,0 +1 @@ +../el8/cloudstack-sccs \ No newline at end of file diff --git a/packaging/suse15/filelimit.conf b/packaging/suse15/filelimit.conf new file mode 120000 index 00000000000..c71688ea640 --- /dev/null +++ b/packaging/suse15/filelimit.conf @@ -0,0 +1 @@ +../el8/filelimit.conf \ No newline at end of file diff --git a/packaging/suse15/replace.properties b/packaging/suse15/replace.properties new file mode 100644 index 00000000000..b1900af8340 --- /dev/null +++ b/packaging/suse15/replace.properties @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +DBUSER=cloud +DBPW=cloud +DBROOTPW= +MSLOG=vmops.log +APISERVERLOG=api.log +DBHOST=localhost +DBDRIVER=jdbc:mysql +COMPONENTS-SPEC=components-premium.xml +REMOTEHOST=localhost +AGENTCLASSPATH= +AGENTLOG=/var/log/cloudstack/agent/agent.log +AGENTLOGDIR=/var/log/cloudstack/agent/ +AGENTSYSCONFDIR=/etc/cloudstack/agent +APISERVERLOG=/var/log/cloudstack/management/apilog.log +BINDIR=/usr/bin +COMMONLIBDIR=/usr/share/cloudstack-common +CONFIGUREVARS= +DEPSCLASSPATH= +DOCDIR= +IPALOCATORLOG=/var/log/cloudstack/management/ipallocator.log +JAVADIR=/usr/share/java +LIBEXECDIR=/usr/libexec +LOCKDIR=/var/lock +MSCLASSPATH= +MSCONF=/etc/cloudstack/management +MSENVIRON=/usr/share/cloudstack-management +MSLOG=/var/log/cloudstack/management/management-server.log +MSLOGDIR=/var/log/cloudstack/management/ +MSMNTDIR=/var/cloudstack/mnt +MSUSER=cloud +PIDDIR=/var/run +PLUGINJAVADIR=/usr/share/cloudstack-management/plugin +PREMIUMJAVADIR=/usr/share/cloudstack-management/premium +PYTHONDIR=/usr/share/cloudstack-common/python-site/ +SERVERSYSCONFDIR=/etc/sysconfig +SETUPDATADIR=/usr/share/cloudstack-management/setup +SYSCONFDIR=/etc/sysconfig +SYSTEMCLASSPATH= +SYSTEMJARS= +USAGECLASSPATH= +USAGELOG=/var/log/cloudstack/usage/usage.log +USAGESYSCONFDIR=/etc/sysconfig +EXTENSIONSDEPLOYMENTMODE=production +GUESTNVRAMTEMPLATELEGACY=/usr/share/qemu/ovmf-x86_64-vars.bin +GUESTLOADERLEGACY=/usr/share/qemu/ovmf-x86_64-code.bin +GUESTNVRAMTEMPLATESECURE=/usr/share/qemu/ovmf-x86_64-ms-vars.bin +GUESTLOADERSECURE=/usr/share/qemu/ovmf-x86_64-ms-code.bin +GUESTNVRAMPATH=/var/lib/libvirt/qemu/nvram/ diff --git a/packaging/systemd/cloudstack-management.default b/packaging/systemd/cloudstack-management.default index 994a1ee8699..a41338beda6 100644 --- a/packaging/systemd/cloudstack-management.default +++ b/packaging/systemd/cloudstack-management.default @@ -17,7 +17,7 @@ JAVA_OPTS="-Djava.security.properties=/etc/cloudstack/management/java.security.ciphers -Djava.awt.headless=true -Xmx2G -XX:+UseParallelGC -XX:MaxGCPauseMillis=500 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/cloudstack/management/ -XX:ErrorFile=/var/log/cloudstack/management/cloudstack-management.err --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED" -CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/java/mysql-connector-java.jar:/usr/share/cloudstack-mysql-ha/lib/*" +CLASSPATH="/usr/share/cloudstack-management/lib/*:/etc/cloudstack/management:/usr/share/cloudstack-common:/usr/share/cloudstack-management/setup:/usr/share/cloudstack-management:/usr/share/cloudstack-mysql-ha/lib/*" BOOTSTRAP_CLASS=org.apache.cloudstack.ServerDaemon diff --git a/packaging/systemd/cloudstack-usage.default b/packaging/systemd/cloudstack-usage.default index 493f40c277a..36b71ac3e0d 100644 --- a/packaging/systemd/cloudstack-usage.default +++ b/packaging/systemd/cloudstack-usage.default @@ -17,7 +17,7 @@ JAVA_OPTS="-Xms256m -Xmx2048m --add-opens=java.base/java.lang=ALL-UNNAMED" -CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage:/usr/share/java/mysql-connector-java.jar" +CLASSPATH="/usr/share/cloudstack-usage/*:/usr/share/cloudstack-usage/lib/*:/usr/share/cloudstack-mysql-ha/lib/*:/etc/cloudstack/usage" JAVA_CLASS=com.cloud.usage.UsageServer diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index c5b690cfcd4..a7121f149e3 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -226,12 +226,12 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } catch (AgentUnavailableException e) { logger.error("Unable to contact backend control plane to initiate backup for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Unable to contact backend control plane to initiate backup"); } catch (OperationTimedoutException e) { logger.error("Operation to initiate backup timed out for VM {}", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); throw new CloudRuntimeException("Operation to initiate backup timed out, please try again"); } @@ -249,12 +249,12 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } else { logger.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); if (answer.getNeedsCleanup()) { - logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state. Backup should be manually deleted to free up the space", vm.getInstanceName()); backupVO.setStatus(Backup.Status.Error); backupDao.update(backupVO.getId(), backupVO); } else { backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + backupDao.update(backupVO.getId(), backupVO); } return new Pair<>(false, null); } @@ -357,7 +357,8 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co volumePools.add(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null); String volumePathPrefix = getVolumePathPrefix(storagePool); - volumePaths.add(String.format("%s/%s", volumePathPrefix, volume.getPath())); + String volumePathSuffix = getVolumePathSuffix(storagePool); + volumePaths.add(String.format("%s%s%s", volumePathPrefix, volume.getPath(), volumePathSuffix)); } return new Pair<>(volumePools, volumePaths); } @@ -367,14 +368,24 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co if (ScopeType.HOST.equals(storagePool.getScope()) || Storage.StoragePoolType.SharedMountPoint.equals(storagePool.getPoolType()) || Storage.StoragePoolType.RBD.equals(storagePool.getPoolType())) { - volumePathPrefix = storagePool.getPath(); + volumePathPrefix = storagePool.getPath() + "/"; + } else if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + volumePathPrefix = "/dev/drbd/by-res/cs-"; } else { // Should be Storage.StoragePoolType.NetworkFilesystem - volumePathPrefix = String.format("/mnt/%s", storagePool.getUuid()); + volumePathPrefix = String.format("/mnt/%s/", storagePool.getUuid()); } return volumePathPrefix; } + private String getVolumePathSuffix(StoragePoolVO storagePool) { + if (Storage.StoragePoolType.Linstor.equals(storagePool.getPoolType())) { + return "/0"; + } else { + return ""; + } + } + @Override public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeInfo backupVolumeInfo, String hostIp, String dataStoreUuid, Pair vmNameAndState) { final VolumeVO volume = volumeDao.findByUuid(backupVolumeInfo.getUuid()); @@ -419,7 +430,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co restoreCommand.setBackupRepoType(backupRepository.getType()); restoreCommand.setBackupRepoAddress(backupRepository.getAddress()); restoreCommand.setVmName(vmNameAndState.first()); - restoreCommand.setRestoreVolumePaths(Collections.singletonList(String.format("%s/%s", getVolumePathPrefix(pool), volumeUUID))); + String restoreVolumePath = String.format("%s%s%s", getVolumePathPrefix(pool), volumeUUID, getVolumePathSuffix(pool)); + restoreCommand.setRestoreVolumePaths(Collections.singletonList(restoreVolumePath)); + restoreCommand.setRestoreVolumeSizes(Collections.singletonList(backedUpVolumeSize)); DataStore dataStore = dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); restoreCommand.setRestoreVolumePools(Collections.singletonList(dataStore != null ? (PrimaryDataStoreTO)dataStore.getTO() : null)); restoreCommand.setDiskType(backupVolumeInfo.getType().name().toLowerCase(Locale.ROOT)); @@ -478,6 +491,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co } else { host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, backup.getZoneId()); } + if (host == null) { + throw new CloudRuntimeException(String.format("Unable to find a running KVM host in zone %d to delete backup %s", backup.getZoneId(), backup.getUuid())); + } DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), backupRepository.getAddress(), backupRepository.getMountOptions()); @@ -564,7 +580,14 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Override public void syncBackupStorageStats(Long zoneId) { final List repositories = backupRepositoryDao.listByZoneAndProvider(zoneId, getName()); + if (CollectionUtils.isEmpty(repositories)) { + return; + } final Host host = resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); + if (host == null) { + logger.warn("Unable to find a running KVM host in zone {} to sync backup storage stats", zoneId); + return; + } for (final BackupRepository repository : repositories) { GetBackupStorageStatsCommand command = new GetBackupStorageStatsCommand(repository.getType(), repository.getAddress(), repository.getMountOptions()); BackupStorageStatsAnswer answer; diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java index 5ff036fef12..d018d488c64 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManager.java @@ -40,17 +40,17 @@ public final class RootCACustomTrustManager implements X509TrustManager { private boolean authStrictness = true; private boolean allowExpiredCertificate = true; private CrlDao crlDao; - private X509Certificate caCertificate; + private List caCertificates; private Map activeCertMap; - public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map activeCertMap, final X509Certificate caCertificate, final CrlDao crlDao) { + public RootCACustomTrustManager(final String clientAddress, final boolean authStrictness, final boolean allowExpiredCertificate, final Map activeCertMap, final List caCertificates, final CrlDao crlDao) { if (StringUtils.isNotEmpty(clientAddress)) { this.clientAddress = clientAddress.replace("/", "").split(":")[0]; } this.authStrictness = authStrictness; this.allowExpiredCertificate = allowExpiredCertificate; this.activeCertMap = activeCertMap; - this.caCertificate = caCertificate; + this.caCertificates = caCertificates; this.crlDao = crlDao; } @@ -151,6 +151,6 @@ public final class RootCACustomTrustManager implements X509TrustManager { @Override public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[]{caCertificate}; + return caCertificates.toArray(new X509Certificate[0]); } } diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java index 25c45ed2a10..afb4f561160 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -40,7 +40,6 @@ import java.security.cert.X509Certificate; import java.security.spec.InvalidKeySpecException; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Enumeration; import java.util.HashSet; import java.util.List; @@ -60,6 +59,7 @@ import org.apache.cloudstack.framework.ca.CAProvider; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.framework.config.ValidatedConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.KeyStoreUtils; @@ -92,6 +92,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con private static KeyPair caKeyPair = null; private static X509Certificate caCertificate = null; + private static List caCertificates = null; private static KeyStore managementKeyStore = null; @Inject @@ -103,20 +104,25 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con /////////////// Root CA Settings /////////////////// //////////////////////////////////////////////////// - private static ConfigKey rootCAPrivateKey = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.private.key", - null, - "The ROOT CA private key.", true); + private static ConfigKey rootCAPrivateKey = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.private.key", null, + "The ROOT CA private key in PEM format. " + + "When set along with the public key and certificate, CloudStack uses this custom CA instead of auto-generating one. " + + "All three ca.plugin.root.* keys must be set together. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validatePrivateKeyPem); - private static ConfigKey rootCAPublicKey = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.public.key", - null, - "The ROOT CA public key.", true); + private static ConfigKey rootCAPublicKey = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.public.key", null, + "The ROOT CA public key in PEM format (X.509/SPKI: must start with '-----BEGIN PUBLIC KEY-----'). " + + "Required when providing a custom CA. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validatePublicKeyPem); - private static ConfigKey rootCACertificate = new ConfigKey<>("Hidden", String.class, - "ca.plugin.root.ca.certificate", - null, - "The ROOT CA certificate.", true); + private static ConfigKey rootCACertificate = new ValidatedConfigKey<>("Hidden", String.class, + "ca.plugin.root.ca.certificate", null, + "The CA certificate(s) in PEM format (must start with '-----BEGIN CERTIFICATE-----'). " + + "For intermediate CAs, concatenate the signing cert first, followed by intermediate(s) and root. " + + "Required when providing a custom CA. Restart management server(s) when changed.", + false, ConfigKey.Scope.Global, null, RootCAProvider::validateCACertificatePem); private static ConfigKey rootCAIssuerDN = new ConfigKey<>("Advanced", String.class, "ca.plugin.root.issuer.dn", @@ -151,7 +157,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con caCertificate, caKeyPair, keyPair.getPublic(), subject, CAManager.CertSignatureAlgorithm.value(), validityDays, domainNames, ipAddresses); - return new Certificate(clientCertificate, keyPair.getPrivate(), Collections.singletonList(caCertificate)); + return new Certificate(clientCertificate, keyPair.getPrivate(), caCertificates); } private Certificate generateCertificateUsingCsr(final String csr, final List names, final List ips, final int validityDays) throws NoSuchAlgorithmException, InvalidKeyException, NoSuchProviderException, CertificateException, SignatureException, IOException, OperatorCreationException { @@ -205,7 +211,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con caCertificate, caKeyPair, request.getPublicKey(), subject, CAManager.CertSignatureAlgorithm.value(), validityDays, dnsNames, ipAddresses); - return new Certificate(clientCertificate, null, Collections.singletonList(caCertificate)); + return new Certificate(clientCertificate, null, caCertificates); } //////////////////////////////////////////////////////// @@ -219,7 +225,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con @Override public List getCaCertificate() { - return Collections.singletonList(caCertificate); + return caCertificates; } @Override @@ -254,8 +260,8 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con private KeyStore getCaKeyStore() throws CertificateException, NoSuchAlgorithmException, IOException, KeyStoreException { final KeyStore ks = KeyStore.getInstance("JKS"); ks.load(null, null); - if (caKeyPair != null && caCertificate != null) { - ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), new X509Certificate[]{caCertificate}); + if (caKeyPair != null && CollectionUtils.isNotEmpty(caCertificates)) { + ks.setKeyEntry(caAlias, caKeyPair.getPrivate(), getKeyStorePassphrase(), caCertificates.toArray(new X509Certificate[0])); } else { return null; } @@ -274,7 +280,7 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con final boolean authStrictness = rootCAAuthStrictness.value(); final boolean allowExpiredCertificate = rootCAAllowExpiredCert.value(); - TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificate, crlDao)}; + TrustManager[] tms = new TrustManager[]{new RootCACustomTrustManager(remoteAddress, authStrictness, allowExpiredCertificate, certMap, caCertificates, crlDao)}; sslContext.init(kmf.getKeyManagers(), tms, new SecureRandom()); final SSLEngine sslEngine = sslContext.createSSLEngine(); @@ -316,33 +322,39 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con if (!configDao.update(rootCAPrivateKey.key(), rootCAPrivateKey.category(), CertUtils.privateKeyToPem(keyPair.getPrivate()))) { logger.error("Failed to save RootCA private key"); } + caKeyPair = keyPair; } catch (final NoSuchProviderException | NoSuchAlgorithmException | IOException e) { logger.error("Failed to generate/save RootCA private/public keys due to exception:", e); } - return loadRootCAKeyPair(); + return caKeyPair != null && caKeyPair.getPrivate() != null && caKeyPair.getPublic() != null; } - private boolean saveNewRootCACertificate() { + boolean saveNewRootCACertificate() { if (caKeyPair == null) { throw new CloudRuntimeException("Cannot issue self-signed root CA certificate as CA keypair is not initialized"); } try { logger.debug("Generating root CA certificate"); - final X509Certificate rootCaCertificate = CertUtils.generateV3Certificate( + final X509Certificate generatedCACert = CertUtils.generateV3Certificate( null, caKeyPair, caKeyPair.getPublic(), rootCAIssuerDN.value(), CAManager.CertSignatureAlgorithm.value(), getCaValidityDays(), null, null); - if (!configDao.update(rootCACertificate.key(), rootCACertificate.category(), CertUtils.x509CertificateToPem(rootCaCertificate))) { + if (!configDao.update(rootCACertificate.key(), rootCACertificate.category(), CertUtils.x509CertificateToPem(generatedCACert))) { logger.error("Failed to update RootCA public/x509 certificate"); } + caCertificates = new ArrayList<>(java.util.Collections.singletonList(generatedCACert)); + caCertificate = generatedCACert; } catch (final CertificateException | NoSuchAlgorithmException | NoSuchProviderException | SignatureException | InvalidKeyException | OperatorCreationException | IOException e) { logger.error("Failed to generate RootCA certificate from private/public keys due to exception:", e); return false; } - return loadRootCACertificate(); + return caCertificate != null; } private boolean loadRootCAKeyPair() { + if (caKeyPair != null) { + return true; + } if (StringUtils.isAnyEmpty(rootCAPublicKey.value(), rootCAPrivateKey.value())) { return false; } @@ -355,14 +367,35 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con return caKeyPair.getPrivate() != null && caKeyPair.getPublic() != null; } - private boolean loadRootCACertificate() { + boolean loadRootCACertificate() { + if (caCertificate != null && CollectionUtils.isNotEmpty(caCertificates)) { + return true; + } + caCertificate = null; + caCertificates = null; if (StringUtils.isEmpty(rootCACertificate.value())) { return false; } try { - caCertificate = CertUtils.pemToX509Certificate(rootCACertificate.value()); - caCertificate.verify(caKeyPair.getPublic()); - } catch (final IOException | CertificateException | NoSuchAlgorithmException | InvalidKeyException | SignatureException | NoSuchProviderException e) { + final List loadedCerts = CertUtils.pemToX509Certificates(rootCACertificate.value()); + if (CollectionUtils.isEmpty(loadedCerts)) { + logger.error("No certificates found in ca.plugin.root.ca.certificate"); + return false; + } + final X509Certificate loadedCACert = loadedCerts.get(0); + + // Verify key ownership without enforcing self-signature + if (!loadedCACert.getPublicKey().equals(caKeyPair.getPublic())) { + logger.error("The public key in the CA certificate does not match the configured CA public key"); + return false; + } + + if (loadedCerts.size() > 1) { + logger.info("Loaded CA certificate chain with {} certificate(s)", loadedCerts.size()); + } + caCertificates = loadedCerts; + caCertificate = loadedCACert; + } catch (final IOException | CertificateException e) { logger.error("Failed to load saved RootCA certificate due to exception:", e); return false; } @@ -389,9 +422,15 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con try { managementKeyStore = KeyStore.getInstance("JKS"); managementKeyStore.load(null, null); - managementKeyStore.setCertificateEntry(caAlias, caCertificate); + int caIndex = 0; + for (final X509Certificate cert : caCertificates) { + managementKeyStore.setCertificateEntry(caAlias + "-" + caIndex++, cert); + } + final List fullChain = new ArrayList<>(); + fullChain.add(serverCertificate.getClientCertificate()); + fullChain.addAll(caCertificates); managementKeyStore.setKeyEntry(managementAlias, serverCertificate.getPrivateKey(), getKeyStorePassphrase(), - new X509Certificate[]{serverCertificate.getClientCertificate(), caCertificate}); + fullChain.toArray(new X509Certificate[0])); } catch (final CertificateException | NoSuchAlgorithmException | KeyStoreException | IOException e) { logger.error("Failed to load root CA management-server keystore due to exception: ", e); return false; @@ -421,14 +460,63 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con } - private boolean setupCA() { - if (!loadRootCAKeyPair() && !saveNewRootCAKeypair()) { - logger.error("Failed to save and load root CA keypair"); - return false; + private static void validatePrivateKeyPem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + CertUtils.pemToPrivateKey(value); + } catch (InvalidKeySpecException | IOException e) { + throw new IllegalArgumentException( + "ca.plugin.root.private.key is not a valid PEM private key: " + e.getMessage()); } - if (!loadRootCACertificate() && !saveNewRootCACertificate()) { - logger.error("Failed to save and load root CA certificate"); - return false; + } + + private static void validatePublicKeyPem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + CertUtils.pemToPublicKey(value); + } catch (InvalidKeySpecException | IOException e) { + throw new IllegalArgumentException( + "ca.plugin.root.public.key is not a valid PEM public key: " + e.getMessage()); + } + } + + static void validateCACertificatePem(String value) { + if (StringUtils.isEmpty(value)) return; + try { + final List certs = CertUtils.pemToX509Certificates(value); + if (CollectionUtils.isEmpty(certs)) { + throw new IllegalArgumentException( + "ca.plugin.root.ca.certificate contains no certificates"); + } + } catch (IOException | CertificateException e) { + throw new IllegalArgumentException( + "ca.plugin.root.ca.certificate is not a valid PEM certificate: " + e.getMessage()); + } + } + + private boolean setupCA() { + if (!loadRootCAKeyPair()) { + if (hasUserProvidedCAKeys()) { + logger.error("Failed to load user-provided CA keys from configuration. " + + "Check that ca.plugin.root.private.key, ca.plugin.root.public.key, and " + + "ca.plugin.root.ca.certificate are all set and in the correct PEM format. " + + "Overwriting with auto-generated keys."); + } + if (!saveNewRootCAKeypair()) { + logger.error("Failed to save and load root CA keypair"); + return false; + } + } + if (!loadRootCACertificate()) { + if (hasUserProvidedCAKeys()) { + logger.error("Failed to load user-provided CA certificate. " + + "Check that ca.plugin.root.ca.certificate is set and in PEM format. " + + "Overwriting with auto-generated certificate."); + } + if (!saveNewRootCACertificate()) { + logger.error("Failed to save and load root CA certificate"); + return false; + } } if (!loadManagementKeyStore()) { logger.error("Failed to check and configure management server keystore"); @@ -437,10 +525,16 @@ public final class RootCAProvider extends AdapterBase implements CAProvider, Con return true; } + private boolean hasUserProvidedCAKeys() { + return StringUtils.isNotEmpty(rootCAPublicKey.value()) + || StringUtils.isNotEmpty(rootCAPrivateKey.value()) + || StringUtils.isNotEmpty(rootCACertificate.value()); + } + @Override public boolean start() { managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value(); - return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore(); + return loadRootCAKeyPair() && loadRootCACertificate() && loadManagementKeyStore(); } @Override diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java index d4ded302332..714e18c3449 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCACustomTrustManagerTest.java @@ -23,9 +23,11 @@ import java.math.BigInteger; import java.security.KeyPair; import java.security.cert.CertificateException; import java.security.cert.X509Certificate; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.List; import org.apache.cloudstack.utils.security.CertUtils; import org.junit.Assert; @@ -63,14 +65,14 @@ public class RootCACustomTrustManagerTest { @Test public void testAuthNotStrictWithInvalidCert() throws Exception { - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(null, null); } @Test public void testAuthNotStrictWithRevokedCert() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(new CrlVO()); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), caCertificate); @@ -79,7 +81,7 @@ public class RootCACustomTrustManagerTest { @Test public void testAuthNotStrictWithInvalidCertOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), caCertificate); @@ -88,14 +90,14 @@ public class RootCACustomTrustManagerTest { @Test(expected = CertificateException.class) public void testAuthNotStrictWithDenyExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, false, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, false, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); } @Test public void testAuthNotStrictWithAllowExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, false, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); Assert.assertTrue(certMap.containsKey(clientIp)); Assert.assertEquals(certMap.get(clientIp), expiredClientCertificate); @@ -103,35 +105,50 @@ public class RootCACustomTrustManagerTest { @Test(expected = CertificateException.class) public void testAuthStrictWithInvalidCert() throws Exception { - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(null, null); } @Test(expected = CertificateException.class) public void testAuthStrictWithRevokedCert() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(new CrlVO()); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); } @Test(expected = CertificateException.class) public void testAuthStrictWithInvalidCertOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{caCertificate}, "RSA"); } @Test(expected = CertificateException.class) public void testAuthStrictWithDenyExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, false, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, false, certMap, Collections.singletonList(caCertificate), crlDao); trustManager.checkClientTrusted(new X509Certificate[]{expiredClientCertificate}, "RSA"); } + @Test + public void testGetAcceptedIssuersWithChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final List chain = Arrays.asList(caCertificate, rootCert); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager( + clientIp, false, true, certMap, chain, crlDao); + + final X509Certificate[] issuers = trustManager.getAcceptedIssuers(); + Assert.assertEquals(2, issuers.length); + Assert.assertEquals(caCertificate, issuers[0]); + Assert.assertEquals(rootCert, issuers[1]); + } + @Test public void testAuthStrictWithAllowExpiredCertAndOwnership() throws Exception { Mockito.when(crlDao.findBySerial(Mockito.any(BigInteger.class))).thenReturn(null); - final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, caCertificate, crlDao); + final RootCACustomTrustManager trustManager = new RootCACustomTrustManager(clientIp, true, true, certMap, Collections.singletonList(caCertificate), crlDao); Assert.assertTrue(trustManager.getAcceptedIssuers() != null); Assert.assertTrue(trustManager.getAcceptedIssuers().length == 1); Assert.assertEquals(trustManager.getAcceptedIssuers()[0], caCertificate); diff --git a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java index 8311f4d45ab..21f00c66a1d 100644 --- a/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java +++ b/plugins/ca/root-ca/src/test/java/org/apache/cloudstack/ca/provider/RootCAProviderTest.java @@ -31,6 +31,7 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.UUID; @@ -38,6 +39,7 @@ import javax.net.ssl.SSLEngine; import org.apache.cloudstack.framework.ca.Certificate; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.utils.security.CertUtils; import org.apache.cloudstack.utils.security.SSLUtils; import org.bouncycastle.asn1.x509.GeneralName; @@ -49,7 +51,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import org.springframework.test.util.ReflectionTestUtils; @RunWith(MockitoJUnitRunner.class) @@ -75,7 +76,7 @@ public class RootCAProviderTest { addField(provider, "caKeyPair", caKeyPair); addField(provider, "caCertificate", caCertificate); - addField(provider, "caKeyPair", caKeyPair); + addField(provider, "caCertificates", Collections.singletonList(caCertificate)); } @After @@ -129,6 +130,46 @@ public class RootCAProviderTest { certificate.getClientCertificate().verify(caCertificate.getPublicKey()); } + @Test + public void testGetCaCertificateWithChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final KeyPair intermediateKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate intermediateCert = CertUtils.generateV3Certificate(rootCert, rootKeyPair, + intermediateKeyPair.getPublic(), "CN=intermediate", "SHA256withRSA", 365, null, null); + + final List chain = Arrays.asList(intermediateCert, rootCert); + addField(provider, "caKeyPair", intermediateKeyPair); + addField(provider, "caCertificate", intermediateCert); + addField(provider, "caCertificates", chain); + + Assert.assertEquals(2, provider.getCaCertificate().size()); + Assert.assertEquals(intermediateCert, provider.getCaCertificate().get(0)); + Assert.assertEquals(rootCert, provider.getCaCertificate().get(1)); + } + + @Test + public void testIssueCertificateWithoutCsrAndChain() throws Exception { + final KeyPair rootKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate rootCert = CertUtils.generateV3Certificate(null, rootKeyPair, rootKeyPair.getPublic(), + "CN=root", "SHA256withRSA", 365, null, null); + final KeyPair intermediateKeyPair = CertUtils.generateRandomKeyPair(1024); + final X509Certificate intermediateCert = CertUtils.generateV3Certificate(rootCert, rootKeyPair, + intermediateKeyPair.getPublic(), "CN=intermediate", "SHA256withRSA", 365, null, null); + + addField(provider, "caKeyPair", intermediateKeyPair); + addField(provider, "caCertificate", intermediateCert); + addField(provider, "caCertificates", Arrays.asList(intermediateCert, rootCert)); + + final Certificate certificate = provider.issueCertificate(Arrays.asList("domain1.com"), null, 1); + Assert.assertNotNull(certificate); + Assert.assertEquals(2, certificate.getCaCertificates().size()); + Assert.assertEquals(intermediateCert, certificate.getCaCertificates().get(0)); + Assert.assertEquals(rootCert, certificate.getCaCertificates().get(1)); + certificate.getClientCertificate().verify(intermediateKeyPair.getPublic()); + } + @Test public void testRevokeCertificate() throws Exception { Assert.assertTrue(provider.revokeCertificate(CertUtils.generateRandomBigInt(), "anyString")); @@ -177,8 +218,8 @@ public class RootCAProviderTest { } @Test - public void testIsManagementCertificateNoMatch() { - ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", "cloudstack"); + public void testIsManagementCertificateNoMatch() throws Exception { + addField(provider, "managementCertificateCustomSAN", "cloudstack"); try { X509Certificate certificate = Mockito.mock(X509Certificate.class); List> altNames = new ArrayList<>(); @@ -193,9 +234,9 @@ public class RootCAProviderTest { } @Test - public void testIsManagementCertificateMatch() { + public void testIsManagementCertificateMatch() throws Exception { String customSAN = "cloudstack"; - ReflectionTestUtils.setField(provider, "managementCertificateCustomSAN", customSAN); + addField(provider, "managementCertificateCustomSAN", customSAN); try { X509Certificate certificate = Mockito.mock(X509Certificate.class); List> altNames = new ArrayList<>(); @@ -208,4 +249,58 @@ public class RootCAProviderTest { Assert.fail(String.format("Exception occurred: %s", e.getMessage())); } } + + @Test + public void testLoadRootCACertificateWithMismatchedCert() throws Exception { + KeyPair otherKeyPair = CertUtils.generateRandomKeyPair(1024); + X509Certificate mismatchedCert = CertUtils.generateV3Certificate(null, otherKeyPair, otherKeyPair.getPublic(), "CN=other", "SHA256withRSA", 365, null, null); + String mismatchedPem = CertUtils.x509CertificateToPem(mismatchedCert); + + ConfigKey mockCertKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockCertKey.value()).thenReturn(mismatchedPem); + addField(provider, "rootCACertificate", mockCertKey); + + addField(provider, "caCertificate", null); + addField(provider, "caCertificates", null); + + Boolean result = provider.loadRootCACertificate(); + Assert.assertFalse(result); + Assert.assertNull(provider.getCaCertificate()); + } + + @Test + public void testSaveNewRootCACertificateWithStaleCache() throws Exception { + ConfigurationDao configDao = Mockito.mock(ConfigurationDao.class); + addField(provider, "configDao", configDao); + + ConfigKey mockCertKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockCertKey.key()).thenReturn("ca.plugin.root.ca.certificate"); + Mockito.when(mockCertKey.category()).thenReturn("Hidden"); + addField(provider, "rootCACertificate", mockCertKey); + + ConfigKey mockIssuerKey = Mockito.mock(ConfigKey.class); + Mockito.when(mockIssuerKey.value()).thenReturn("CN=ca.cloudstack.apache.org"); + addField(provider, "rootCAIssuerDN", mockIssuerKey); + + addField(provider, "caCertificate", null); + addField(provider, "caCertificates", null); + + Mockito.when(configDao.update(Mockito.anyString(), Mockito.anyString(), Mockito.anyString())).thenReturn(true); + + Boolean result = provider.saveNewRootCACertificate(); + Assert.assertTrue(result); + Assert.assertNotNull(provider.getCaCertificate()); + Assert.assertEquals(1, provider.getCaCertificate().size()); + } + + @Test + public void testValidateCACertificatePem() throws Exception { + String truncatedPem = "-----BEGIN CERTIFICATE-----\nMIICxTCCAa0CAQAw\n"; + try { + RootCAProvider.validateCACertificatePem(truncatedPem); + Assert.fail("Expected IllegalArgumentException"); + } catch (IllegalArgumentException e) { + Assert.assertTrue(e.getMessage().contains("is not a valid PEM certificate")); + } + } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java index 0cec0df6618..e5c89c5de1c 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaBalanceCmd.java @@ -17,12 +17,9 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; -import com.cloud.user.Account; - import org.apache.cloudstack.api.ACL; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.ApiConstants; @@ -30,36 +27,41 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.api.response.QuotaStatementItemResponse; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaBalance", responseObject = QuotaStatementItemResponse.class, description = "Create a quota balance statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, +@APICommand(name = "quotaBalance", responseObject = QuotaBalanceResponse.class, description = "Create Quota balance statements for the provided Accounts or Projects.", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaBalanceCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "Name of the Account for which balance statements will be generated. " + + "Deprecated, please use 'accountid' instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "ID of the domain which the Account identified by 'account' belongs to." + + "Deprecated, please use 'accountid' instead.") private Long domainId; - @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "End of the period of the Quota balance." + - ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) + @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, description = "Date of the last Quota balance to be returned. Must be informed together with the " + + "startdate parameter, and can not be before startdate. " + ApiConstants.PARAMETER_DESCRIPTION_END_DATE_POSSIBLE_FORMATS) private Date endDate; - @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Start of the period of the Quota balance. " + + @Parameter(name = ApiConstants.START_DATE, type = CommandType.DATE, description = "Date of the first Quota balance to be returned. Must be before today. " + ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "ID of the Account for which balance statements will be generated.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "ID of the Project for which balance statements will be generated. Can not be specified with accountid.") + private Long projectId; + @Inject QuotaResponseBuilder _responseBuilder; @@ -88,48 +90,32 @@ public class QuotaBalanceCmd extends BaseCmd { } public Date getEndDate() { - if (endDate == null){ - return null; - } - else{ - return _responseBuilder.startOfNextDay(new Date(endDate.getTime())); - } + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; + if (ObjectUtils.allNull(accountId, accountName, domainId, projectId)) { + return -1; } - Account account = _accountService.getActiveAccountByName(accountName, domainId); - if (account != null) { - return account.getAccountId(); - } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaBalance(this); - - QuotaBalanceResponse response; - if (endDate == null) { - response = _responseBuilder.createQuotaLastBalanceResponse(quotaUsage, getStartDate()); - } else { - response = _responseBuilder.createQuotaBalanceResponse(quotaUsage, getStartDate(), new Date(endDate.getTime())); - } + QuotaBalanceResponse response = _responseBuilder.createQuotaBalanceResponse(this); response.setResponseName(getCommandName()); setResponseObject(response); } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java index d3bd3868ed1..bfe26a9f425 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/command/QuotaStatementCmd.java @@ -17,7 +17,6 @@ package org.apache.cloudstack.api.command; import java.util.Date; -import java.util.List; import javax.inject.Inject; @@ -28,24 +27,25 @@ import org.apache.cloudstack.api.BaseCmd; import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.response.AccountResponse; import org.apache.cloudstack.api.response.DomainResponse; +import org.apache.cloudstack.api.response.ProjectResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementItemResponse; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; -import com.cloud.user.Account; +import org.apache.commons.lang3.ObjectUtils; -@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a quota statement", since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, - httpMethod = "GET") +@APICommand(name = "quotaStatement", responseObject = QuotaStatementItemResponse.class, description = "Create a Quota statement for the provided Account, Project, or Domain.", + since = "4.7.0", requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, httpMethod = "GET") public class QuotaStatementCmd extends BaseCmd { - - - @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, required = true, description = "Optional, Account Id for which statement needs to be generated") + @ACL + @Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, + description = "Name of the Account for which the Quota statement will be generated. Deprecated, please use accountid instead.") private String accountName; @ACL - @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, required = true, entityType = DomainResponse.class, description = "Optional, If domain Id is given and the caller is domain admin then the statement is generated for domain.") + @Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, + description = "ID of the Domain for which the Quota statement will be generated. May be used individually or with account.") private Long domainId; @Parameter(name = ApiConstants.END_DATE, type = CommandType.DATE, required = true, description = "End of the period of the Quota statement. " + @@ -56,15 +56,25 @@ public class QuotaStatementCmd extends BaseCmd { ApiConstants.PARAMETER_DESCRIPTION_START_DATE_POSSIBLE_FORMATS) private Date startDate; - @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, description = "List quota usage records for the specified usage type") + @Parameter(name = ApiConstants.TYPE, type = CommandType.INTEGER, + description = "Consider only Quota usage records for the specified usage type in the statement.") private Integer usageType; @ACL - @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, description = "List usage records for the specified Account") + @Parameter(name = ApiConstants.ACCOUNT_ID, type = CommandType.UUID, entityType = AccountResponse.class, + description = "ID of the Account for which the Quota statement will be generated. Can not be specified with projectid.") private Long accountId; + @ACL + @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, + description = "ID of the Project for which the Quota statement will be generated. Can not be specified with accountid.", since = "4.23.0") + private Long projectId; + + @Parameter(name = ApiConstants.SHOW_RESOURCES, type = CommandType.BOOLEAN, description = "List the resources of each Quota type in the period.", since = "4.23.0") + private boolean showResources; + @Inject - private QuotaResponseBuilder _responseBuilder; + QuotaResponseBuilder responseBuilder; public Long getAccountId() { return accountId; @@ -99,43 +109,47 @@ public class QuotaStatementCmd extends BaseCmd { } public Date getEndDate() { - return _responseBuilder.startOfNextDay(endDate == null ? new Date() : new Date(endDate.getTime())); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + this.endDate = endDate; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; + } + + public boolean isShowResources() { + return showResources; + } + + public void setShowResources(boolean showResources) { + this.showResources = showResources; + } + + public Long getProjectId() { + return projectId; } @Override public long getEntityOwnerId() { - if (accountId != null) { - return accountId; + if (ObjectUtils.allNull(accountId, accountName, projectId)) { + return -1; } - Account activeAccountByName = _accountService.getActiveAccountByName(accountName, domainId); - if (activeAccountByName != null) { - return activeAccountByName.getAccountId(); - } - return Account.ACCOUNT_ID_SYSTEM; + return _accountService.finalizeAccountId(accountId, accountName, domainId, projectId); } @Override public void execute() { - List quotaUsage = _responseBuilder.getQuotaUsage(this); - - QuotaStatementResponse response = _responseBuilder.createQuotaStatementResponse(quotaUsage); - response.setStartDate(startDate == null ? null : new Date(startDate.getTime())); - response.setEndDate(endDate == null ? null : new Date(endDate.getTime())); - + QuotaStatementResponse response = responseBuilder.createQuotaStatementResponse(this); + response.setStartDate(startDate); + response.setEndDate(endDate); response.setResponseName(getCommandName()); setResponseObject(response); } - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java index 625f4829c67..714066b048a 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaBalanceResponse.java @@ -17,137 +17,65 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; import java.util.Date; import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import com.cloud.serializer.Param; public class QuotaBalanceResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; - - @SerializedName("startquota") - @Param(description = "Quota started with") - private BigDecimal startQuota; - - @SerializedName("endquota") - @Param(description = "Quota by end of this period") - private BigDecimal endQuota; - - @SerializedName("credits") - @Param(description = "List of credits made during this period") - private List credits = null; - - @SerializedName("startdate") - @Param(description = "Start date") - private Date startDate = null; - - @SerializedName("enddate") - @Param(description = "End date") - private Date endDate = null; - - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Balance's currency.") private String currency; + @SerializedName(ApiConstants.DATE) + @Param(description = "Balance's date.") + private Date date; + + @SerializedName(ApiConstants.BALANCE) + @Param(description = "Balance's value.") + private BigDecimal balance; + + @SerializedName(ApiConstants.BALANCES) + @Param(description = "List of balances in the period.") + private List balances; + public QuotaBalanceResponse() { - super(); - credits = new ArrayList(); + super("balance"); } - public Long getAccountId() { - return accountId; + public QuotaBalanceResponse(Date date, BigDecimal balance) { + super("balance"); + this.date = date; + this.balance = balance; } - public void setAccountId(Long accountId) { - this.accountId = accountId; + public void setCurrency(String currency) { + this.currency = currency; } - public String getAccountName() { - return accountName; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - - public BigDecimal getStartQuota() { - return startQuota; - } - - public void setStartQuota(BigDecimal startQuota) { - this.startQuota = startQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public BigDecimal getEndQuota() { - return endQuota; - } - - public void setEndQuota(BigDecimal endQuota) { - this.endQuota = endQuota.setScale(2, RoundingMode.HALF_EVEN); - } - - public List getCredits() { - return credits; - } - - public void setCredits(List credits) { - this.credits = credits; - } - - public void addCredits(QuotaBalanceVO credit) { - QuotaCreditsResponse cr = new QuotaCreditsResponse(); - cr.setCredit(credit.getCreditBalance()); - cr.setCreditedOn(credit.getUpdatedOn()); - credits.add(0, cr); - } - - public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); - } - - public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); - } - - public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); - } - - public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); + public void setBalances(List balances) { + this.balances = balances; } public String getCurrency() { return currency; } - public void setCurrency(String currency) { - this.currency = currency; + public List getBalances() { + return balances; + } + + public Date getDate() { + return date; + } + + public BigDecimal getBalance() { + return balance; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java index 177fb00d4b5..f0390bf626d 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilder.java @@ -29,10 +29,8 @@ import org.apache.cloudstack.api.command.QuotaTariffCreateCmd; import org.apache.cloudstack.api.command.QuotaTariffListCmd; import org.apache.cloudstack.api.command.QuotaTariffUpdateCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import java.util.Date; import java.util.List; @@ -49,18 +47,12 @@ public interface QuotaResponseBuilder { boolean isUserAllowedToSeeActivationRules(User user); - QuotaStatementResponse createQuotaStatementResponse(List quotaUsage); + QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd); - QuotaBalanceResponse createQuotaBalanceResponse(List quotaUsage, Date startDate, Date endDate); + QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd); Pair, Integer> createQuotaSummaryResponse(QuotaSummaryCmd cmd); - QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate); - - List getQuotaUsage(QuotaStatementCmd cmd); - - List getQuotaBalance(QuotaBalanceCmd cmd); - QuotaCreditsResponse addQuotaCredits(Long accountId, Long domainId, Double amount, Long updatedBy, Boolean enforce); List listQuotaEmailTemplates(QuotaEmailTemplateListCmd cmd); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java index d1992d4c3ad..231ba777e60 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImpl.java @@ -21,21 +21,17 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.lang.reflect.ParameterizedType; import java.math.BigDecimal; -import java.math.RoundingMode; import java.time.LocalDate; import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; -import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.Date; import java.util.HashMap; -import java.util.Iterator; import java.util.List; -import java.util.ListIterator; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -43,12 +39,37 @@ import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.domain.Domain; +import com.cloud.domain.DomainVO; +import com.cloud.domain.dao.DomainDao; +import com.cloud.event.ActionEvent; +import com.cloud.event.EventTypes; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.PermissionDeniedException; -import com.cloud.projects.dao.ProjectDao; +import com.cloud.network.dao.IPAddressDao; +import com.cloud.network.dao.IPAddressVO; +import com.cloud.network.dao.NetworkDao; +import com.cloud.network.dao.NetworkVO; +import com.cloud.offerings.dao.NetworkOfferingDao; +import com.cloud.offerings.NetworkOfferingVO; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.SnapshotVO; +import com.cloud.user.Account; +import com.cloud.user.AccountManager; +import com.cloud.user.AccountVO; +import com.cloud.user.dao.AccountDao; +import com.cloud.user.dao.UserDao; import com.cloud.user.User; import com.cloud.user.UserVO; import com.cloud.utils.DateUtil; +import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.ApiErrorCode; import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.command.QuotaBalanceCmd; @@ -74,6 +95,7 @@ import org.apache.cloudstack.quota.activationrule.presetvariables.ComputingResou import org.apache.cloudstack.quota.activationrule.presetvariables.GenericPresetVariable; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariableDefinition; import org.apache.cloudstack.quota.activationrule.presetvariables.PresetVariables; +import org.apache.cloudstack.quota.activationrule.presetvariables.ResourceCounting; import org.apache.cloudstack.quota.activationrule.presetvariables.Value; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.constant.QuotaTypes; @@ -93,7 +115,8 @@ import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaSummaryVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; +import org.apache.cloudstack.quota.vo.QuotaUsageResourceVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils; import org.apache.commons.collections4.CollectionUtils; @@ -105,18 +128,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; -import com.cloud.domain.DomainVO; -import com.cloud.domain.dao.DomainDao; -import com.cloud.event.ActionEvent; -import com.cloud.event.EventTypes; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.user.Account; -import com.cloud.user.AccountManager; -import com.cloud.user.AccountVO; -import com.cloud.user.dao.AccountDao; -import com.cloud.user.dao.UserDao; -import com.cloud.utils.Pair; - @Component public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { protected Logger logger = LogManager.getLogger(getClass()); @@ -139,8 +150,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { @Inject private AccountDao _accountDao; @Inject - private ProjectDao projectDao; - @Inject private QuotaAccountDao quotaAccountDao; @Inject private DomainDao domainDao; @@ -158,17 +167,26 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { private JsInterpreterHelper jsInterpreterHelper; @Inject private ApiDiscoveryService apiDiscoveryService; + @Inject + private IPAddressDao ipAddressDao; + @Inject + private NetworkDao networkDao; + @Inject + private NetworkOfferingDao networkOfferingDao; + @Inject + private SnapshotDao snapshotDao; + @Inject + private VMInstanceDao vmInstanceDao; + @Inject + private VMTemplateDao vmTemplateDao; + @Inject + private VolumeDao volumeDao; - private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class}; + + private final Class[] assignableClasses = {GenericPresetVariable.class, ComputingResources.class, ResourceCounting.class}; private Set accountTypesThatCanListAllQuotaSummaries = Sets.newHashSet(Account.Type.ADMIN, Account.Type.DOMAIN_ADMIN); - protected void checkActivationRulesAllowed(String activationRule) { - if (!_quotaService.isJsInterpretationEnabled() && StringUtils.isNotEmpty(activationRule)) { - throw new PermissionDeniedException("Quota Tariff Activation Rule cannot be set, as Javascript interpretation is disabled in the configuration."); - } - } - @Override public QuotaTariffResponse createQuotaTariffResponse(QuotaTariffVO tariff, boolean returnActivationRule) { final QuotaTariffResponse response = new QuotaTariffResponse(); @@ -308,168 +326,227 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } @Override - public QuotaBalanceResponse createQuotaBalanceResponse(List quotaBalance, Date startDate, Date endDate) { - if (quotaBalance == null || quotaBalance.isEmpty()) { - throw new InvalidParameterValueException("The request period does not contain balance entries."); - } - Collections.sort(quotaBalance, new Comparator() { - @Override - public int compare(QuotaBalanceVO o1, QuotaBalanceVO o2) { - o1 = o1 == null ? new QuotaBalanceVO() : o1; - o2 = o2 == null ? new QuotaBalanceVO() : o2; - return o2.getUpdatedOn().compareTo(o1.getUpdatedOn()); // desc - } - }); + public QuotaBalanceResponse createQuotaBalanceResponse(QuotaBalanceCmd cmd) { + List quotaBalances = _quotaService.listQuotaBalancesForAccount(cmd.getEntityOwnerId(), cmd.getStartDate(), cmd.getEndDate()); - boolean have_balance_entries = false; - //check that there is at least one balance entry - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (entry.isBalanceEntry()) { - have_balance_entries = true; - break; - } - } - //if last entry is a credit deposit then remove that as that is already - //accounted for in the starting balance after that entry, note the sort is desc - if (have_balance_entries) { - ListIterator li = quotaBalance.listIterator(quotaBalance.size()); - // Iterate in reverse. - while (li.hasPrevious()) { - QuotaBalanceVO entry = li.previous(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - li.remove(); - } else { - break; - } - } - } + List balances = quotaBalances.stream() + .map(balance -> new QuotaBalanceResponse(balance.getUpdatedOn(), balance.getCreditBalance())) + .collect(Collectors.toList()); - int quota_activity = quotaBalance.size(); - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - boolean consecutive = true; - for (Iterator it = quotaBalance.iterator(); it.hasNext();) { - QuotaBalanceVO entry = it.next(); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: All Credit Entry=" + entry); - } - if (entry.getCreditsId() > 0) { - if (consecutive) { - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.addCredits(entry); - it.remove(); - } else { - consecutive = false; - } - } + QuotaBalanceResponse response = new QuotaBalanceResponse(); + response.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); + response.setBalances(balances); - if (quota_activity > 0 && quotaBalance.size() > 0) { - // order is desc last item is the start item - QuotaBalanceVO startItem = quotaBalance.get(quotaBalance.size() - 1); - QuotaBalanceVO endItem = quotaBalance.get(0); - resp.setStartDate(startDate); - resp.setStartQuota(startItem.getCreditBalance()); - resp.setEndDate(endDate); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaBalanceResponse: Start Entry=" + startItem); - logger.debug("createQuotaBalanceResponse: End Entry=" + endItem); - } - resp.setEndQuota(endItem.getCreditBalance().add(lastCredits)); - } else if (quota_activity > 0) { - // order is desc last item is the start item - resp.setStartDate(startDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndDate(endDate); - resp.setEndQuota(new BigDecimal(0).add(lastCredits)); - } else { - resp.setStartDate(startDate); - resp.setEndDate(endDate); - resp.setStartQuota(new BigDecimal(0)); - resp.setEndQuota(new BigDecimal(0)); - } - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; + return response; } @Override - public QuotaStatementResponse createQuotaStatementResponse(final List quotaUsage) { - if (quotaUsage == null || quotaUsage.isEmpty()) { - throw new InvalidParameterValueException("There is no usage data found for period mentioned."); - } + public QuotaStatementResponse createQuotaStatementResponse(QuotaStatementCmd cmd) { + Long accountId = getAccountIdForQuotaStatement(cmd); + Long domainId = getDomainIdForQuotaStatement(cmd, accountId); + List quotaUsages = _quotaService.getQuotaUsage(accountId, null, domainId, cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); + + logger.debug("Creating quota statement from [{}] usage records for parameters [{}].", quotaUsages.size(), + ReflectionToStringBuilderUtils.reflectOnlySelectedFields(cmd, "accountName", "accountId", "projectId", "domainId", "startDate", "endDate", "usageType", "showResources")); + createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(quotaUsages, cmd.getUsageType()); + + Map> recordsPerUsageTypes = quotaUsages.stream() + .sorted(Comparator.comparingInt(QuotaUsageJoinVO::getUsageType)) + .collect(Collectors.groupingBy(QuotaUsageJoinVO::getUsageType)); + + List items = new ArrayList<>(); + recordsPerUsageTypes.forEach((key, value) -> items.add(createStatementItem(key, value, cmd.isShowResources()))); QuotaStatementResponse statement = new QuotaStatementResponse(); - - HashMap quotaTariffMap = new HashMap(); - Collection result = QuotaTypes.listQuotaTypes().values(); - - for (QuotaTypes quotaTariff : result) { - quotaTariffMap.put(quotaTariff.getQuotaType(), quotaTariff); - // add dummy record for each usage type - QuotaUsageVO dummy = new QuotaUsageVO(quotaUsage.get(0)); - dummy.setUsageType(quotaTariff.getQuotaType()); - dummy.setQuotaUsed(new BigDecimal(0)); - quotaUsage.add(dummy); - } - - if (logger.isDebugEnabled()) { - logger.debug( - "createQuotaStatementResponse Type=" + quotaUsage.get(0).getUsageType() + " usage=" + quotaUsage.get(0).getQuotaUsed().setScale(2, RoundingMode.HALF_EVEN) - + " rec.id=" + quotaUsage.get(0).getUsageItemId() + " SD=" + quotaUsage.get(0).getStartDate() + " ED=" + quotaUsage.get(0).getEndDate()); - } - - Collections.sort(quotaUsage, new Comparator() { - @Override - public int compare(QuotaUsageVO o1, QuotaUsageVO o2) { - if (o1.getUsageType() == o2.getUsageType()) { - return 0; - } - return o1.getUsageType() < o2.getUsageType() ? -1 : 1; - } - }); - - List items = new ArrayList(); - QuotaStatementItemResponse lineitem; - int type = -1; - BigDecimal usage = new BigDecimal(0); - BigDecimal totalUsage = new BigDecimal(0); - quotaUsage.add(new QuotaUsageVO());// boundary - QuotaUsageVO prev = quotaUsage.get(0); - if (logger.isDebugEnabled()) { - logger.debug("createQuotaStatementResponse record count=" + quotaUsage.size()); - } - for (final QuotaUsageVO quotaRecord : quotaUsage) { - if (type != quotaRecord.getUsageType()) { - if (type != -1) { - lineitem = new QuotaStatementItemResponse(type); - lineitem.setQuotaUsed(usage); - lineitem.setAccountId(prev.getAccountId()); - lineitem.setDomainId(prev.getDomainId()); - lineitem.setUsageUnit(quotaTariffMap.get(type).getQuotaUnit()); - lineitem.setUsageName(quotaTariffMap.get(type).getQuotaName()); - lineitem.setObjectName("quotausage"); - items.add(lineitem); - totalUsage = totalUsage.add(usage); - usage = new BigDecimal(0); - } - type = quotaRecord.getUsageType(); - } - prev = quotaRecord; - usage = usage.add(quotaRecord.getQuotaUsed()); - } - statement.setLineItem(items); - statement.setTotalQuota(totalUsage); + statement.setTotalQuota(items.stream().map(QuotaStatementItemResponse::getQuotaUsed).reduce(BigDecimal.ZERO, BigDecimal::add)); statement.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); statement.setObjectName("statement"); + + if (accountId != null) { + Account account = _accountDao.findByIdIncludingRemoved(accountId); + statement.setAccountId(account.getUuid()); + statement.setAccountName(account.getAccountName()); + domainId = account.getDomainId(); + } + if (domainId != null) { + DomainVO domain = domainDao.findByIdIncludingRemoved(domainId); + statement.setDomainId(domain.getUuid()); + } + return statement; } + protected Long getAccountIdForQuotaStatement(QuotaStatementCmd cmd) { + if (Account.Type.NORMAL.equals(CallContext.current().getCallingAccount().getType())) { + logger.debug("Limiting the Quota statement for the calling Account, as they are a User Account."); + return CallContext.current().getCallingAccountId(); + } + + long accountId = cmd.getEntityOwnerId(); + if (accountId != -1) { + return accountId; + } + + if (cmd.getDomainId() == null) { + logger.debug("Limiting the Quota statement for the calling Account, as 'domainid' was not informed."); + return CallContext.current().getCallingAccountId(); + } + + logger.debug("Allowing admin/domain admin to generate the Quota statement for the provided Domain."); + return null; + } + + protected Long getDomainIdForQuotaStatement(QuotaStatementCmd cmd, Long accountId) { + if (accountId != null) { + logger.debug("Quota statement is already limited to Account [{}].", accountId); + Account account = _accountDao.findByIdIncludingRemoved(accountId); + return account.getDomainId(); + } + + Long domainId = cmd.getDomainId(); + if (domainId != null) { + return domainId; + } + + logger.debug("Limiting the Quota statement for the calling Account's Domain."); + return CallContext.current().getCallingAccount().getDomainId(); + } + + protected void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(List quotaUsages, Integer usageType) { + if (usageType != null) { + logger.debug("As the usage type [{}] was informed as parameter of the API quotaStatement, we will not create dummy records.", usageType); + return; + + } + + for (Integer quotaType : QuotaTypes.listQuotaTypes().keySet()) { + QuotaUsageJoinVO dummy = new QuotaUsageJoinVO(); + dummy.setUsageType(quotaType); + dummy.setQuotaUsed(BigDecimal.ZERO); + quotaUsages.add(dummy); + } + } + + protected QuotaStatementItemResponse createStatementItem(int usageType, List usageRecords, boolean showResources) { + QuotaUsageJoinVO firstRecord = usageRecords.get(0); + int type = firstRecord.getUsageType(); + + QuotaTypes quotaType = QuotaTypes.listQuotaTypes().get(type); + + QuotaStatementItemResponse item = new QuotaStatementItemResponse(type); + item.setQuotaUsed(usageRecords.stream().map(QuotaUsageJoinVO::getQuotaUsed).filter(Objects::nonNull).reduce(BigDecimal.ZERO, BigDecimal::add)); + item.setUsageUnit(quotaType.getQuotaUnit()); + item.setUsageName(quotaType.getQuotaName()); + + setStatementItemResources(item, usageType, usageRecords, showResources); + return item; + } + + protected void setStatementItemResources(QuotaStatementItemResponse statementItem, int usageType, List quotaUsageRecords, boolean showResources) { + if (!showResources) { + return; + } + + List itemDetails = new ArrayList<>(); + + Map quotaUsagesValuesAggregatedById = quotaUsageRecords + .stream() + .filter(quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType) != null) + .collect(Collectors.groupingBy( + quotaUsageJoinVo -> getResourceIdByUsageType(quotaUsageJoinVo, usageType), + Collectors.reducing(new BigDecimal(0), QuotaUsageJoinVO::getQuotaUsed, BigDecimal::add) + )); + + for (Map.Entry entry : quotaUsagesValuesAggregatedById.entrySet()) { + QuotaStatementItemResourceResponse detail = new QuotaStatementItemResourceResponse(); + + detail.setQuotaUsed(entry.getValue()); + + QuotaUsageResourceVO resource = getResourceFromIdAndType(entry.getKey(), usageType); + if (resource != null) { + detail.setResourceId(resource.getUuid()); + detail.setDisplayName(resource.getName()); + detail.setRemoved(resource.isRemoved()); + } else { + detail.setDisplayName(""); + + } + itemDetails.add(detail); + } + statementItem.setResources(itemDetails); + } + + protected Long getResourceIdByUsageType(QuotaUsageJoinVO quotaUsageJoinVo, int usageType) { + switch (usageType) { + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + return quotaUsageJoinVo.getNetworkId(); + case QuotaTypes.NETWORK_OFFERING: + return quotaUsageJoinVo.getOfferingId(); + default: + return quotaUsageJoinVo.getResourceId(); + } + } + + protected QuotaUsageResourceVO getResourceFromIdAndType(long resourceId, int usageType) { + switch (usageType) { + case QuotaTypes.ALLOCATED_VM: + case QuotaTypes.RUNNING_VM: + VMInstanceVO vmInstance = vmInstanceDao.findByIdIncludingRemoved(resourceId); + if (vmInstance != null) { + return new QuotaUsageResourceVO(vmInstance.getUuid(), vmInstance.getHostName(), vmInstance.getRemoved()); + } + break; + case QuotaTypes.VOLUME: + case QuotaTypes.VOLUME_SECONDARY: + case QuotaTypes.VM_DISK_BYTES_READ: + case QuotaTypes.VM_DISK_BYTES_WRITE: + case QuotaTypes.VM_DISK_IO_READ: + case QuotaTypes.VM_DISK_IO_WRITE: + VolumeVO volume = volumeDao.findByIdIncludingRemoved(resourceId); + if (volume != null) { + return new QuotaUsageResourceVO(volume.getUuid(), volume.getName(), volume.getRemoved()); + } + break; + case QuotaTypes.VM_SNAPSHOT_ON_PRIMARY: + case QuotaTypes.VM_SNAPSHOT: + case QuotaTypes.SNAPSHOT: + SnapshotVO snapshot = snapshotDao.findByIdIncludingRemoved(resourceId); + if (snapshot != null) { + return new QuotaUsageResourceVO(snapshot.getUuid(), snapshot.getName(), snapshot.getRemoved()); + } + break; + case QuotaTypes.NETWORK_BYTES_SENT: + case QuotaTypes.NETWORK_BYTES_RECEIVED: + NetworkVO network = networkDao.findByIdIncludingRemoved(resourceId); + if (network != null) { + return new QuotaUsageResourceVO(network.getUuid(), network.getName(), network.getRemoved()); + } + break; + case QuotaTypes.TEMPLATE: + case QuotaTypes.ISO: + VMTemplateVO vmTemplate = vmTemplateDao.findByIdIncludingRemoved(resourceId); + if (vmTemplate != null) { + return new QuotaUsageResourceVO(vmTemplate.getUuid(), vmTemplate.getName(), vmTemplate.getRemoved()); + } + break; + case QuotaTypes.NETWORK_OFFERING: + NetworkOfferingVO networkOffering = networkOfferingDao.findByIdIncludingRemoved(resourceId); + if (networkOffering != null) { + return new QuotaUsageResourceVO(networkOffering.getUuid(), networkOffering.getName(), networkOffering.getRemoved()); + } + break; + case QuotaTypes.IP_ADDRESS: + IPAddressVO ipAddress = ipAddressDao.findByIdIncludingRemoved(resourceId); + if (ipAddress != null) { + return new QuotaUsageResourceVO(ipAddress.getUuid(), ipAddress.getName(), ipAddress.getRemoved()); + } + break; + } + return null; + } + @Override public Pair, Integer> listQuotaTariffPlans(final QuotaTariffListCmd cmd) { Date startDate = cmd.getEffectiveDate(); @@ -501,6 +578,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { Integer position = cmd.getPosition(); warnQuotaTariffUpdateDeprecatedFields(cmd); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); @@ -508,8 +586,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { throw new InvalidParameterValueException(String.format("There is no quota tariffs with name [%s].", name)); } - checkActivationRulesAllowed(activationRule); - Date currentQuotaTariffStartDate = currentQuotaTariff.getEffectiveOn(); currentQuotaTariff.setRemoved(now); @@ -524,14 +600,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { } protected void warnQuotaTariffUpdateDeprecatedFields(QuotaTariffUpdateCmd cmd) { - String warnMessage = "The parameter 's%s' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; + String warnMessage = "The parameter '{}' for API 'quotaTariffUpdate' is no longer needed and it will be removed in future releases."; if (cmd.getStartDate() != null) { - logger.warn(String.format(warnMessage,"startdate")); + logger.warn(warnMessage, "startdate"); } if (cmd.getUsageType() != null) { - logger.warn(String.format(warnMessage,"usagetype")); + logger.warn(warnMessage, "usagetype"); } } @@ -630,7 +706,7 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { throw new InvalidParameterValueException("Account does not exist with account id " + accountId); } final boolean lockAccountEnforcement = "true".equalsIgnoreCase(QuotaConfig.QuotaEnableEnforcement.value()); - final BigDecimal currentAccountBalance = _quotaBalanceDao.lastQuotaBalance(accountId, domainId, startOfNextDay(new Date(depositedOn.getTime()))); + final BigDecimal currentAccountBalance = _quotaBalanceDao.getLastQuotaBalance(accountId, domainId); logger.debug("Depositing [{}] credits on adjusted date [{}]; current balance is [{}].", amount, DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), depositedOn), currentAccountBalance); // update quota account with the balance @@ -694,39 +770,6 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { return false; } - @Override - public QuotaBalanceResponse createQuotaLastBalanceResponse(List quotaBalance, Date startDate) { - if (quotaBalance == null) { - throw new InvalidParameterValueException("There are no balance entries on or before the requested date."); - } - if (startDate == null) { - startDate = new Date(); - } - QuotaBalanceResponse resp = new QuotaBalanceResponse(); - BigDecimal lastCredits = new BigDecimal(0); - for (QuotaBalanceVO entry : quotaBalance) { - logger.debug("createQuotaLastBalanceResponse Date={} balance={} credit={}", - DateUtil.displayDateInTimezone(QuotaManagerImpl.getUsageAggregationTimeZone(), entry.getUpdatedOn()), - entry.getCreditBalance(), entry.getCreditsId()); - - lastCredits = lastCredits.add(entry.getCreditBalance()); - } - resp.setStartQuota(lastCredits); - resp.setStartDate(startDate); - resp.setCurrency(QuotaConfig.QuotaCurrencySymbol.value()); - resp.setObjectName("balance"); - return resp; - } - - @Override - public List getQuotaUsage(QuotaStatementCmd cmd) { - return _quotaService.getQuotaUsage(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getUsageType(), cmd.getStartDate(), cmd.getEndDate()); - } - - @Override - public List getQuotaBalance(QuotaBalanceCmd cmd) { - return _quotaService.findQuotaBalanceVO(cmd.getAccountId(), cmd.getAccountName(), cmd.getDomainId(), cmd.getStartDate(), cmd.getEndDate()); - } @Override public Date startOfNextDay(Date date) { LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); @@ -758,14 +801,14 @@ public class QuotaResponseBuilderImpl implements QuotaResponseBuilder { String activationRule = cmd.getActivationRule(); Integer position = ObjectUtils.defaultIfNull(cmd.getPosition(), 1); + jsInterpreterHelper.ensureInterpreterEnabledIfParameterProvided(ApiConstants.ACTIVATION_RULE, StringUtils.isNotBlank(activationRule)); + QuotaTariffVO currentQuotaTariff = _quotaTariffDao.findByName(name); if (currentQuotaTariff != null) { throw new InvalidParameterValueException(String.format("A quota tariff with name [%s] already exist.", name)); } - checkActivationRulesAllowed(activationRule); - if (startDate.compareTo(now) < 0) { throw new InvalidParameterValueException(String.format("The value passed as Quota tariff's start date is in the past: [%s]. " + "Please, inform a date in the future or do not pass the parameter to use the current date and time.", startDate)); diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java new file mode 100644 index 00000000000..3e052f73339 --- /dev/null +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResourceResponse.java @@ -0,0 +1,61 @@ +//Licensed to the Apache Software Foundation (ASF) under one +//or more contributor license agreements. See the NOTICE file +//distributed with this work for additional information +//regarding copyright ownership. The ASF licenses this file +//to you under the Apache License, Version 2.0 (the +//"License"); you may not use this file except in compliance +//with the License. You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, +//software distributed under the License is distributed on an +//"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +//KIND, either express or implied. See the License for the +//specific language governing permissions and limitations +//under the License. +package org.apache.cloudstack.api.response; + +import java.math.BigDecimal; + +import com.google.gson.annotations.SerializedName; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import com.cloud.serializer.Param; + +public class QuotaStatementItemResourceResponse extends BaseResponse { + + @SerializedName(ApiConstants.QUOTA_CONSUMED) + @Param(description = "Quota consumed.") + private BigDecimal quotaUsed; + + @SerializedName(ApiConstants.RESOURCE_ID) + @Param(description = "Resources's ID.") + private String resourceId; + + @SerializedName(ApiConstants.DISPLAY_NAME) + @Param(description = "Resource's display name.") + private String displayName; + + @SerializedName(ApiConstants.REMOVED) + @Param(description = "Indicates whether the resource is removed or active.") + private boolean removed; + + public void setQuotaUsed(BigDecimal quotaUsed) { + this.quotaUsed = quotaUsed; + } + + public void setResourceId(String resourceId) { + this.resourceId = resourceId; + } + + public void setDisplayName(String displayName) { + this.displayName = displayName; + } + + public void setRemoved(boolean removed) { + this.removed = removed; + } +} diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java index c370d82b3cc..0747c5a9172 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementItemResponse.java @@ -17,72 +17,41 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; -import java.math.RoundingMode; +import java.util.List; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; public class QuotaStatementItemResponse extends BaseResponse { - @SerializedName("type") - @Param(description = "Usage type") + @SerializedName(ApiConstants.TYPE) + @Param(description = "Usage type.") private int usageType; - @SerializedName("accountid") - @Param(description = "Account id") - private Long accountId; - - @SerializedName("account") - @Param(description = "Account name") - private String accountName; - - @SerializedName("domain") - @Param(description = "Domain id") - private Long domainId; - - @SerializedName("name") - @Param(description = "Usage type name") + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the Usage type.") private String usageName; - @SerializedName("unit") - @Param(description = "Usage unit") + @SerializedName(ApiConstants.UNIT) + @Param(description = "Unit of the Usage type.") private String usageUnit; - @SerializedName("quota") - @Param(description = "Quota consumed") + @SerializedName(ApiConstants.QUOTA) + @Param(description = "Quota consumed.") private BigDecimal quotaUsed; + @SerializedName(ApiConstants.RESOURCES) + @Param(description = "Item's resources.") + private List resources; + public QuotaStatementItemResponse(final int usageType) { this.usageType = usageType; } - public Long getAccountId() { - return accountId; - } - - public void setAccountId(Long accountId) { - this.accountId = accountId; - } - - public String getAccountName() { - return accountName; - } - - public void setAccountName(String accountName) { - this.accountName = accountName; - } - - public Long getDomainId() { - return domainId; - } - - public void setDomainId(Long domainId) { - this.domainId = domainId; - } - public String getUsageName() { return usageName; } @@ -112,7 +81,15 @@ public class QuotaStatementItemResponse extends BaseResponse { } public void setQuotaUsed(BigDecimal quotaUsed) { - this.quotaUsed = quotaUsed.setScale(2, RoundingMode.HALF_EVEN); + this.quotaUsed = quotaUsed; + } + + public List getResources() { + return resources; + } + + public void setResources(List resources) { + this.resources = resources; } } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java index 0a7ba4dbeb9..81cb1011182 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/api/response/QuotaStatementResponse.java @@ -18,56 +18,56 @@ package org.apache.cloudstack.api.response; import com.cloud.serializer.Param; import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import java.math.BigDecimal; -import java.math.RoundingMode; import java.util.Date; import java.util.List; public class QuotaStatementResponse extends BaseResponse { - @SerializedName("accountid") - @Param(description = "Account ID") - private Long accountId; + @SerializedName(ApiConstants.ACCOUNT_ID) + @Param(description = "ID of the Account.") + private String accountId; - @SerializedName("account") - @Param(description = "Account name") + @SerializedName(ApiConstants.ACCOUNT) + @Param(description = "Name of the Account.") private String accountName; - @SerializedName("domain") - @Param(description = "Domain ID") - private Long domainId; + @SerializedName(ApiConstants.DOMAIN) + @Param(description = "ID of the Domain.") + private String domainId; - @SerializedName("quotausage") - @Param(description = "List of quota usage under various types", responseObject = QuotaStatementItemResponse.class) + @SerializedName(ApiConstants.QUOTA_USAGE) + @Param(description = "List of Quota usage under various types.", responseObject = QuotaStatementItemResponse.class) private List lineItem; - @SerializedName("totalquota") - @Param(description = "Total quota used during this period") + @SerializedName(ApiConstants.TOTAL_QUOTA) + @Param(description = "Total Quota consumed during this period.") private BigDecimal totalQuota; - @SerializedName("startdate") - @Param(description = "Start date") + @SerializedName(ApiConstants.START_DATE) + @Param(description = "Start date of the Quota statement.") private Date startDate = null; - @SerializedName("enddate") - @Param(description = "End date") + @SerializedName(ApiConstants.END_DATE) + @Param(description = "End date of the Quota statement.") private Date endDate = null; - @SerializedName("currency") - @Param(description = "Currency") + @SerializedName(ApiConstants.CURRENCY) + @Param(description = "Currency of the Quota statement.") private String currency; public QuotaStatementResponse() { super(); } - public Long getAccountId() { + public String getAccountId() { return accountId; } - public void setAccountId(Long accountId) { + public void setAccountId(String accountId) { this.accountId = accountId; } @@ -79,45 +79,36 @@ public class QuotaStatementResponse extends BaseResponse { this.accountName = accountName; } - public Long getDomainId() { + public String getDomainId() { return domainId; } - public void setDomainId(Long domainId) { + public void setDomainId(String domainId) { this.domainId = domainId; } - public List getLineItem() { - return lineItem; - } - public void setLineItem(List lineItem) { this.lineItem = lineItem; } public Date getStartDate() { - return startDate == null ? null : new Date(startDate.getTime()); + return startDate; } public void setStartDate(Date startDate) { - this.startDate = startDate == null ? null : new Date(startDate.getTime()); + this.startDate = startDate; } public Date getEndDate() { - return endDate == null ? null : new Date(endDate.getTime()); + return endDate; } public void setEndDate(Date endDate) { - this.endDate = endDate == null ? null : new Date(endDate.getTime()); - } - - - public BigDecimal getTotalQuota() { - return totalQuota; + this.endDate = endDate; } public void setTotalQuota(BigDecimal totalQuota) { - this.totalQuota = totalQuota.setScale(2, RoundingMode.HALF_EVEN); + this.totalQuota = totalQuota; } public String getCurrency() { diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java index f6a34e01be8..478e43d2e20 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaService.java @@ -21,16 +21,16 @@ import java.util.Date; import java.util.List; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import com.cloud.user.AccountVO; import com.cloud.utils.component.PluggableService; public interface QuotaService extends PluggableService { - List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); + List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate); - List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate); + List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate); void setLockAccount(Long accountId, Boolean state); @@ -40,6 +40,4 @@ public interface QuotaService extends PluggableService { boolean saveQuotaAccount(AccountVO account, BigDecimal aggrUsage, Date endDate); - boolean isJsInterpretationEnabled(); - } diff --git a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java index f39153ae0ac..e1ec2ebb2a6 100644 --- a/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java +++ b/plugins/database/quota/src/main/java/org/apache/cloudstack/quota/QuotaServiceImpl.java @@ -28,6 +28,7 @@ import javax.naming.ConfigurationException; import com.cloud.projects.ProjectManager; import com.cloud.user.AccountService; +import com.cloud.utils.DateUtil; import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsCmd; @@ -53,23 +54,21 @@ import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.quota.constant.QuotaConfig; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; -import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.time.DateUtils; import org.springframework.stereotype.Component; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; import com.cloud.exception.InvalidParameterValueException; -import com.cloud.exception.PermissionDeniedException; -import com.cloud.server.ManagementService; import com.cloud.user.Account; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.component.ManagerBase; -import com.cloud.utils.db.Filter; @Component public class QuotaServiceImpl extends ManagerBase implements QuotaService, Configurable, QuotaConfig { @@ -81,7 +80,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi @Inject private QuotaAccountDao _quotaAcc; @Inject - private QuotaUsageDao _quotaUsageDao; + private QuotaUsageJoinDao quotaUsageJoinDao; @Inject private DomainDao _domainDao; @Inject @@ -95,8 +94,6 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi private TimeZone _usageTimezone; - private boolean jsInterpretationEnabled = false; - public QuotaServiceImpl() { super(); } @@ -108,8 +105,6 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi String timeZoneStr = ObjectUtils.defaultIfNull(_configDao.getValue(Config.UsageAggregationTimezone.toString()), "GMT"); _usageTimezone = TimeZone.getTimeZone(timeZoneStr); - jsInterpretationEnabled = ManagementService.JsInterpretationEnabled.value(); - return true; } @@ -157,88 +152,58 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi } @Override - public List findQuotaBalanceVO(Long accountId, String accountName, Long domainId, Date startDate, Date endDate) { - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); + public List listQuotaBalancesForAccount(Long accountId, Date startDate, Date endDate) { + validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + + if (accountId == -1) { + accountId = CallContext.current().getCallingAccountId(); + } + Account account = _accountDao.findByIdIncludingRemoved(accountId); + Long domainId = account.getDomainId(); + + if (startDate == null && endDate == null) { + logger.debug("Retrieving last quota balance for {}.", account); + QuotaBalanceVO lastQuotaBalance = _quotaBalanceDao.getLastQuotaBalanceEntry(accountId, domainId, null); + + if (lastQuotaBalance == null) { + logger.debug("Did not found a quota balance entry for {}.", account); + return new ArrayList<>(); } + + return List.of(lastQuotaBalance); } - startDate = startDate == null ? new Date() : startDate; - if (endDate == null) { - // adjust start date to end of day as there is no end date - startDate = _respBldr.startOfNextDay(startDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance1: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", on or before " + startDate); - } - List qbrecords = _quotaBalanceDao.lastQuotaBalanceVO(accountId, domainId, startDate); - if (logger.isDebugEnabled()) { - logger.debug("Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("Incorrect Date there are no quota records before this date " + startDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - if (startDate.before(endDate)) { - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance2: Getting quota balance records for account: " + accountId + ", domainId: " + domainId + ", between " + startDate - + " and " + endDate); - } - List qbrecords = _quotaBalanceDao.findQuotaBalance(accountId, domainId, startDate, endDate); - if (logger.isDebugEnabled()) { - logger.debug("getQuotaBalance3: Found records size=" + qbrecords.size()); - } - if (qbrecords.isEmpty()) { - logger.info("There are no quota records between these dates start date " + startDate + " and end date:" + endDate); - return qbrecords; - } else { - return qbrecords; - } - } else { - throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); - } + endDate = DateUtils.addDays(new Date(), -1); + } + + List quotaBalances = _quotaBalanceDao.listQuotaBalances(accountId, domainId, startDate, endDate); + + if (quotaBalances.isEmpty()) { + logger.info("There are no quota balances for {} between [{}] and [{}].", account, + DateUtil.getOutputString(startDate), DateUtil.getOutputString(endDate)); + } + + return quotaBalances; + } + + protected void validateStartDateAndEndDateForListQuotaBalancesForAccount(Date startDate, Date endDate) { + if (startDate == null && endDate != null) { + throw new InvalidParameterValueException("Parameter \"enddate\" must be informed together with parameter \"startdate\"."); + } + + Date now = new Date(); + if (startDate != null && startDate.after(now)) { + throw new InvalidParameterValueException("The last balance can be at most from yesterday; therefore, the start date must be before today."); + } + + if (ObjectUtils.allNotNull(startDate, endDate) && startDate.after(endDate)) { + throw new InvalidParameterValueException("The start date cannot be after the end date."); } } @Override - public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { - // if accountId is not specified, use accountName and domainId - if ((accountId == null) && (accountName != null) && (domainId != null)) { - Account userAccount = null; - Account caller = CallContext.current().getCallingAccount(); - if (_domainDao.isChildDomain(caller.getDomainId(), domainId)) { - Filter filter = new Filter(AccountVO.class, "id", Boolean.FALSE, null, null); - List accounts = _accountDao.listAccounts(accountName, domainId, filter); - if (!accounts.isEmpty()) { - userAccount = accounts.get(0); - } - if (userAccount != null) { - accountId = userAccount.getId(); - } else { - throw new InvalidParameterValueException("Unable to find account " + accountName + " in domain " + domainId); - } - } else { - throw new PermissionDeniedException("Invalid Domain Id or Account"); - } - } - + public List getQuotaUsage(Long accountId, String accountName, Long domainId, Integer usageType, Date startDate, Date endDate) { if (startDate.after(endDate)) { throw new InvalidParameterValueException("Incorrect Date Range. Start date: " + startDate + " is after end date:" + endDate); } @@ -246,7 +211,7 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi logger.debug("Getting quota records of type [{}] for account [{}] in domain [{}], between [{}] and [{}].", usageType, accountId, domainId, startDate, endDate); - return _quotaUsageDao.findQuotaUsage(accountId, domainId, usageType, startDate, endDate); + return quotaUsageJoinDao.findQuotaUsage(accountId, domainId, usageType, null, null, null, startDate, endDate, null); } @Override @@ -298,9 +263,4 @@ public class QuotaServiceImpl extends ManagerBase implements QuotaService, Confi _quotaAcc.updateQuotaAccount(accountId, acc); } } - - @Override - public boolean isJsInterpretationEnabled() { - return jsInterpretationEnabled; - } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java index adabc694f25..cd0eeb2c3b4 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaBalanceCmdTest.java @@ -16,18 +16,15 @@ // under the License. package org.apache.cloudstack.api.command; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - import org.apache.cloudstack.api.response.QuotaBalanceResponse; import org.apache.cloudstack.api.response.QuotaResponseBuilder; -import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; import junit.framework.TestCase; @@ -36,31 +33,20 @@ import junit.framework.TestCase; public class QuotaBalanceCmdTest extends TestCase { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder quotaResponseBuilderMock; + + @InjectMocks + @Spy + QuotaBalanceCmd quotaBalanceCmdSpy; @Test - public void testQuotaBalanceCmd() throws NoSuchFieldException, IllegalAccessException { - QuotaBalanceCmd cmd = new QuotaBalanceCmd(); - Field rbField = QuotaBalanceCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + public void executeTestSetResponseObject() { + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + Mockito.doReturn(expected).when(quotaResponseBuilderMock).createQuotaBalanceResponse(Mockito.eq(quotaBalanceCmdSpy)); - List quotaBalanceVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaBalance(Mockito.any(cmd.getClass()))).thenReturn(quotaBalanceVOList); - Mockito.when(responseBuilder.createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.when(responseBuilder.createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(new QuotaBalanceResponse()); - Mockito.lenient().when(responseBuilder.startOfNextDay(Mockito.any(Date.class))).thenReturn(new Date()); + quotaBalanceCmdSpy.execute(); - // end date not specified - cmd.setStartDate(new Date()); - cmd.setEndDate(null); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaLastBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class)); - Mockito.verify(responseBuilder, Mockito.times(0)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); - - // end date specified - cmd.setEndDate(new Date()); - cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).createQuotaBalanceResponse(Mockito.eq(quotaBalanceVOList), Mockito.any(Date.class), Mockito.any(Date.class)); + Assert.assertEquals(expected, quotaBalanceCmdSpy.getResponseObject()); } + } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java index d6f9f747fa8..e67638db632 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/command/QuotaStatementCmdTest.java @@ -16,38 +16,29 @@ // under the License. package org.apache.cloudstack.api.command; -import junit.framework.TestCase; import org.apache.cloudstack.api.response.QuotaResponseBuilder; import org.apache.cloudstack.api.response.QuotaStatementResponse; -import org.apache.cloudstack.quota.vo.QuotaUsageVO; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.List; - @RunWith(MockitoJUnitRunner.class) -public class QuotaStatementCmdTest extends TestCase { +public class QuotaStatementCmdTest { @Mock - QuotaResponseBuilder responseBuilder; + QuotaResponseBuilder responseBuilderMock; @Test - public void testQuotaStatementCmd() throws NoSuchFieldException, IllegalAccessException { + public void executeTestVerifyCalls() { QuotaStatementCmd cmd = new QuotaStatementCmd(); cmd.setAccountName("admin"); + cmd.responseBuilder = responseBuilderMock; - Field rbField = QuotaStatementCmd.class.getDeclaredField("_responseBuilder"); - rbField.setAccessible(true); - rbField.set(cmd, responseBuilder); + Mockito.doReturn(new QuotaStatementResponse()).when(responseBuilderMock).createQuotaStatementResponse(Mockito.any()); - List quotaUsageVOList = new ArrayList(); - Mockito.when(responseBuilder.getQuotaUsage(Mockito.eq(cmd))).thenReturn(quotaUsageVOList); - Mockito.when(responseBuilder.createQuotaStatementResponse(Mockito.eq(quotaUsageVOList))).thenReturn(new QuotaStatementResponse()); cmd.execute(); - Mockito.verify(responseBuilder, Mockito.times(1)).getQuotaUsage(Mockito.eq(cmd)); + + Mockito.verify(responseBuilderMock).createQuotaStatementResponse(cmd); } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java index ea88a106b84..0c7610461c1 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/api/response/QuotaResponseBuilderImplTest.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.api.response; import java.math.BigDecimal; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -38,10 +40,12 @@ import com.cloud.user.UserVO; import com.cloud.utils.Pair; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.command.QuotaBalanceCmd; import org.apache.cloudstack.api.command.QuotaConfigureEmailCmd; import org.apache.cloudstack.api.command.QuotaCreditsListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateListCmd; import org.apache.cloudstack.api.command.QuotaEmailTemplateUpdateCmd; +import org.apache.cloudstack.api.command.QuotaStatementCmd; import org.apache.cloudstack.api.command.QuotaSummaryCmd; import org.apache.cloudstack.api.command.QuotaValidateActivationRuleCmd; import org.apache.cloudstack.context.CallContext; @@ -67,6 +71,7 @@ import org.apache.cloudstack.quota.vo.QuotaCreditsVO; import org.apache.cloudstack.quota.vo.QuotaEmailConfigurationVO; import org.apache.cloudstack.quota.vo.QuotaEmailTemplatesVO; import org.apache.cloudstack.quota.vo.QuotaTariffVO; +import org.apache.cloudstack.quota.vo.QuotaUsageJoinVO; import org.apache.cloudstack.utils.jsinterpreter.JsInterpreter; import org.apache.commons.lang3.time.DateUtils; @@ -245,7 +250,7 @@ public class QuotaResponseBuilderImplTest extends TestCase { credit.setCredit(new BigDecimal(amount)); Mockito.when(quotaCreditsDaoMock.saveCredits(Mockito.any(QuotaCreditsVO.class))).thenReturn(credit); - Mockito.when(quotaBalanceDaoMock.lastQuotaBalance(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(Date.class))).thenReturn(new BigDecimal(111)); + Mockito.when(quotaBalanceDaoMock.getLastQuotaBalance(Mockito.anyLong(), Mockito.anyLong())).thenReturn(new BigDecimal(111)); Mockito.doReturn(userVoMock).when(quotaResponseBuilderSpy).getCreditorForQuotaCredits(credit); AccountVO account = new AccountVO(); @@ -291,33 +296,6 @@ public class QuotaResponseBuilderImplTest extends TestCase { assertTrue(quotaResponseBuilderSpy.updateQuotaEmailTemplate(cmd)); } - @Test - public void testCreateQuotaLastBalanceResponse() { - List quotaBalance = new ArrayList<>(); - // null balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(null, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // empty balance test - try { - quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, new Date()); - } catch (InvalidParameterValueException e) { - assertTrue(e.getMessage().equals("There are no balance entries on or before the requested date.")); - } - - // valid balance test - QuotaBalanceVO entry = new QuotaBalanceVO(); - entry.setAccountId(2L); - entry.setCreditBalance(new BigDecimal(100)); - quotaBalance.add(entry); - quotaBalance.add(entry); - QuotaBalanceResponse resp = quotaResponseBuilderSpy.createQuotaLastBalanceResponse(quotaBalance, null); - assertTrue(resp.getStartQuota().compareTo(new BigDecimal(200)) == 0); - } - @Test public void testStartOfNextDayWithoutParameters() { Date nextDate = quotaResponseBuilderSpy.startOfNextDay(); @@ -914,4 +892,239 @@ public class QuotaResponseBuilderImplTest extends TestCase { Assert.assertTrue(formattedVariables.containsValue("accountname")); Assert.assertTrue(formattedVariables.containsValue("zonename")); } + + private List getQuotaBalancesForTest() { + List balances = new ArrayList<>(); + + QuotaBalanceVO balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-10.42)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-18.94)); + balances.add(balance); + + balance = new QuotaBalanceVO(); + balance.setUpdatedOn(new Date()); + balance.setCreditBalance(BigDecimal.valueOf(-29.37)); + balances.add(balance); + + return balances; + } + + @Test + public void createQuotaBalancesResponseTestCreateResponse() { + List balances = getQuotaBalancesForTest(); + + QuotaBalanceResponse expected = new QuotaBalanceResponse(); + expected.setObjectName("balance"); + expected.setCurrency("$"); + + Mockito.doReturn(balances).when(quotaServiceMock).listQuotaBalancesForAccount(Mockito.any(), Mockito.any(), Mockito.any()); + QuotaBalanceResponse result = quotaResponseBuilderSpy.createQuotaBalanceResponse(new QuotaBalanceCmd()); + + Assert.assertEquals(expected.getCurrency(), result.getCurrency()); + + for (int i = 0; i < balances.size(); i++) { + Assert.assertEquals(balances.get(i).getUpdatedOn(), result.getBalances().get(i).getDate()); + Assert.assertEquals(balances.get(i).getCreditBalance(), result.getBalances().get(i).getBalance()); + } + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeDifferentFromNullDoNothing() { + List listUsage = new ArrayList<>(); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, 1); + + Assert.assertTrue(listUsage.isEmpty()); + } + + @Test + public void createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformedTestUsageTypeIsNullAddDummyForAllQuotaTypes() { + List listUsage = new ArrayList<>(); + listUsage.add(new QuotaUsageJoinVO()); + + quotaResponseBuilderSpy.createDummyRecordForEachQuotaTypeIfUsageTypeIsNotInformed(listUsage, null); + + Assert.assertEquals(QuotaTypes.listQuotaTypes().size() + 1, listUsage.size()); + + QuotaTypes.listQuotaTypes().entrySet().forEach(entry -> { + Assert.assertTrue(listUsage.stream().anyMatch(usage -> usage.getUsageType() == entry.getKey() && usage.getQuotaUsed().equals(BigDecimal.ZERO))); + }); + } + + private List getQuotaUsagesForTest() { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); + + List quotaUsages = new ArrayList<>(); + + QuotaUsageJoinVO quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(1l); + quotaUsage.setDomainId(2l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(10)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-01")); + quotaUsage.setEndDate(sdf.parse("2022-01-02")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(4l); + quotaUsage.setDomainId(5l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(null); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-03")); + quotaUsage.setEndDate(sdf.parse("2022-01-04")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + quotaUsage = new QuotaUsageJoinVO(); + quotaUsage.setAccountId(6l); + quotaUsage.setDomainId(7l); + quotaUsage.setUsageType(3); + quotaUsage.setQuotaUsed(BigDecimal.valueOf(5)); + try { + quotaUsage.setStartDate(sdf.parse("2022-01-05")); + quotaUsage.setEndDate(sdf.parse("2022-01-06")); + } catch (ParseException ignored) { + } + quotaUsages.add(quotaUsage); + + return quotaUsages; + } + + @Test + public void createStatementItemTestReturnItem() { + List quotaUsages = getQuotaUsagesForTest(); + Mockito.doNothing().when(quotaResponseBuilderSpy).setStatementItemResources(Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyBoolean()); + + QuotaStatementItemResponse result = quotaResponseBuilderSpy.createStatementItem(0, quotaUsages, false); + + QuotaUsageJoinVO expected = quotaUsages.get(0); + QuotaTypes quotaTypeExpected = QuotaTypes.listQuotaTypes().get(expected.getUsageType()); + Assert.assertEquals(BigDecimal.valueOf(15), result.getQuotaUsed()); + Assert.assertEquals(quotaTypeExpected.getQuotaUnit(), result.getUsageUnit()); + Assert.assertEquals(quotaTypeExpected.getQuotaName(), result.getUsageName()); + } + + @Test + public void setStatementItemResourcesTestDoNotShowResourcesDoNothing() { + QuotaStatementItemResponse item = new QuotaStatementItemResponse(1); + + quotaResponseBuilderSpy.setStatementItemResources(item, 0, getQuotaUsagesForTest(), false); + + Assert.assertNull(item.getResources()); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountForNormalUser() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.NORMAL).when(accountMock).getType(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsEntityOwnerIdWhenProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(42L).when(cmd).getEntityOwnerId(); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(42L), result); + } + + @Test + public void getAccountIdForQuotaStatementTestLimitsToCallingAccountWhenCallerIsAdminAndDomainIsNotProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(null).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertEquals(Long.valueOf(callerAccountMock.getAccountId()), result); + } + } + + @Test + public void getAccountIdForQuotaStatementTestReturnsNullWhenCallerIsAdminAndDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(accountMock).when(callContextMock).getCallingAccount(); + Mockito.doReturn(Account.Type.ADMIN).when(accountMock).getType(); + Mockito.doReturn(-1L).when(cmd).getEntityOwnerId(); + Mockito.doReturn(10L).when(cmd).getDomainId(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getAccountIdForQuotaStatement(cmd); + + Assert.assertNull(result); + } + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsAccountDomainIdWhenAccountIdIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + AccountVO account = Mockito.mock(AccountVO.class); + + Mockito.doReturn(account).when(accountDaoMock).findByIdIncludingRemoved(55L); + Mockito.doReturn(77L).when(account).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, 55L); + + Assert.assertEquals(Long.valueOf(77L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestReturnsProvidedDomainIdWhenAccountIdIsNull() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + + Mockito.doReturn(99L).when(cmd).getDomainId(); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(Long.valueOf(99L), result); + } + + @Test + public void getDomainIdForQuotaStatementTestFallsBackToCallingAccountDomainIdWhenNeitherAccountNorDomainIsProvided() { + QuotaStatementCmd cmd = Mockito.mock(QuotaStatementCmd.class); + Account account = Mockito.mock(Account.class); + + Mockito.doReturn(null).when(cmd).getDomainId(); + Mockito.doReturn(123L).when(account).getDomainId(); + Mockito.doReturn(account).when(callContextMock).getCallingAccount(); + + try (MockedStatic callContextMocked = Mockito.mockStatic(CallContext.class)) { + callContextMocked.when(CallContext::current).thenReturn(callContextMock); + + Long result = quotaResponseBuilderSpy.getDomainIdForQuotaStatement(cmd, null); + + Assert.assertEquals(123L, result.longValue()); + } + } } diff --git a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java index 259264f3b0e..89c48b1cc66 100644 --- a/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java +++ b/plugins/database/quota/src/test/java/org/apache/cloudstack/quota/QuotaServiceImplTest.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.quota; import com.cloud.configuration.Config; import com.cloud.domain.dao.DomainDao; +import com.cloud.exception.InvalidParameterValueException; import com.cloud.user.AccountVO; import com.cloud.user.dao.AccountDao; import com.cloud.utils.db.TransactionLegacy; @@ -28,9 +29,12 @@ import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.QuotaAccountDao; import org.apache.cloudstack.quota.dao.QuotaBalanceDao; import org.apache.cloudstack.quota.dao.QuotaUsageDao; +import org.apache.cloudstack.quota.dao.QuotaUsageJoinDao; import org.apache.cloudstack.quota.vo.QuotaAccountVO; import org.apache.cloudstack.quota.vo.QuotaBalanceVO; +import org.apache.commons.lang3.time.DateUtils; import org.joda.time.DateTime; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -42,7 +46,6 @@ import org.mockito.junit.MockitoJUnitRunner; import javax.naming.ConfigurationException; import java.lang.reflect.Field; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -50,6 +53,8 @@ import java.util.List; @RunWith(MockitoJUnitRunner.class) public class QuotaServiceImplTest extends TestCase { + @Mock + AccountVO accountVoMock; @Mock AccountDao accountDaoMock; @Mock @@ -63,10 +68,9 @@ public class QuotaServiceImplTest extends TestCase { @Mock QuotaBalanceDao quotaBalanceDao; @Mock - QuotaResponseBuilder respBldr; + QuotaUsageJoinDao quotaUsageJoinDaoMock; @Mock - private AccountVO accountVoMock; - + QuotaResponseBuilder respBldr; @Spy @InjectMocks QuotaServiceImpl quotaServiceImplSpy; @@ -85,9 +89,9 @@ public class QuotaServiceImplTest extends TestCase { quotaAccountDaoField.setAccessible(true); quotaAccountDaoField.set(quotaServiceImplSpy, quotaAcc); - Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("_quotaUsageDao"); + Field quotaUsageDaoField = QuotaServiceImpl.class.getDeclaredField("quotaUsageJoinDao"); quotaUsageDaoField.setAccessible(true); - quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageDao); + quotaUsageDaoField.set(quotaServiceImplSpy, quotaUsageJoinDaoMock); Field domainDaoField = QuotaServiceImpl.class.getDeclaredField("_domainDao"); domainDaoField.setAccessible(true); @@ -109,30 +113,6 @@ public class QuotaServiceImplTest extends TestCase { quotaServiceImplSpy.configure("randomName", null); } - @Test - public void testFindQuotaBalanceVO() { - final long accountId = 2L; - final String accountName = "admin123"; - final long domainId = 1L; - final Date startDate = new DateTime().minusDays(2).toDate(); - final Date endDate = new Date(); - - List records = new ArrayList<>(); - QuotaBalanceVO qb = new QuotaBalanceVO(); - qb.setCreditBalance(new BigDecimal(100)); - qb.setAccountId(accountId); - records.add(qb); - - Mockito.when(respBldr.startOfNextDay(Mockito.any(Date.class))).thenReturn(startDate); - Mockito.when(quotaBalanceDao.findQuotaBalance(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class), Mockito.any(Date.class))).thenReturn(records); - Mockito.when(quotaBalanceDao.lastQuotaBalanceVO(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.any(Date.class))).thenReturn(records); - - // with enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, endDate).get(0).equals(qb)); - // without enddate - assertTrue(quotaServiceImplSpy.findQuotaBalanceVO(accountId, accountName, domainId, startDate, null).get(0).equals(qb)); - } - @Test public void testGetQuotaUsage() { final long accountId = 2L; @@ -142,7 +122,8 @@ public class QuotaServiceImplTest extends TestCase { final Date endDate = new Date(); quotaServiceImplSpy.getQuotaUsage(accountId, accountName, domainId, QuotaTypes.IP_ADDRESS, startDate, endDate); - Mockito.verify(quotaUsageDao, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(Date.class), Mockito.any(Date.class)); + Mockito.verify(quotaUsageJoinDaoMock, Mockito.times(1)).findQuotaUsage(Mockito.eq(accountId), Mockito.eq(domainId), Mockito.eq(QuotaTypes.IP_ADDRESS), Mockito.any(), + Mockito.any(), Mockito.any(), Mockito.any(Date.class), Mockito.any(Date.class), Mockito.any()); } @Test @@ -178,4 +159,66 @@ public class QuotaServiceImplTest extends TestCase { Mockito.verify(quotaAcc, Mockito.times(1)).persistQuotaAccount(Mockito.any(QuotaAccountVO.class)); } + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsNullAndEndDateIsNotNullThrowsInvalidParameterException() { + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(null, new Date()); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterNowThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, null); + } + + @Test + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestEndDateIsAfterNowDoesNothing() { + Date startDate = DateUtils.addMinutes(new Date(), -1); + Date endDate = DateUtils.addMinutes(new Date(), 1); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test(expected = InvalidParameterValueException.class) + public void validateStartDateAndEndDateForListQuotaBalancesForAccountTestStartDateIsAfterEndDateThrowsInvalidParameterValueException() { + Date startDate = DateUtils.addMinutes(new Date(), -10); + Date endDate = DateUtils.addMinutes(new Date(), -15); + quotaServiceImplSpy.validateStartDateAndEndDateForListQuotaBalancesForAccount(startDate, endDate); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNullReturnsEmptyList() { + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(null).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void listQuotaBalancesForAccountTestLastQuotaBalanceIsNotNullReturnsIt() { + QuotaBalanceVO expected = new QuotaBalanceVO(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).getLastQuotaBalanceEntry(Mockito.anyLong(), Mockito.anyLong(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, null, null); + + Assert.assertEquals(expected, result.get(0)); + } + + @Test + public void listQuotaBalancesForAccountTestReturnsQuotaBalances() { + List expected = new ArrayList<>(); + + Mockito.doNothing().when(quotaServiceImplSpy).validateStartDateAndEndDateForListQuotaBalancesForAccount(Mockito.any(), Mockito.any()); + Mockito.doReturn(expected).when(quotaBalanceDao).listQuotaBalances(Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.any()); + Mockito.doReturn(Mockito.mock(AccountVO.class)).when(accountDaoMock).findByIdIncludingRemoved(Mockito.anyLong()); + + List result = quotaServiceImplSpy.listQuotaBalancesForAccount(1L, new Date(), null); + + Assert.assertEquals(expected, result); + } + } diff --git a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java index ac840c00be3..3f2d85458d3 100644 --- a/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java +++ b/plugins/event-bus/webhook/src/main/java/org/apache/cloudstack/mom/webhook/WebhookDeliveryThread.java @@ -48,6 +48,8 @@ import org.apache.http.HttpHeaders; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpPost; +import javax.net.ssl.SSLContext; + import org.apache.http.conn.ssl.NoopHostnameVerifier; import org.apache.http.conn.ssl.TrustAllStrategy; import org.apache.http.entity.ContentType; @@ -97,7 +99,9 @@ public class WebhookDeliveryThread implements Runnable { protected void setHttpClient() throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { if (webhook.isSslVerification()) { - httpClient = HttpClients.createDefault(); + httpClient = HttpClients.custom() + .setSSLContext(SSLContext.getDefault()) + .build(); return; } httpClient = HttpClients diff --git a/plugins/host-allocators/random/pom.xml b/plugins/host-allocators/random/pom.xml deleted file mode 100644 index 061caf1573f..00000000000 --- a/plugins/host-allocators/random/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - cloud-plugin-host-allocator-random - Apache CloudStack Plugin - Host Allocator Random - - org.apache.cloudstack - cloudstack-plugins - 4.23.0.0-SNAPSHOT - ../../pom.xml - - diff --git a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java b/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java deleted file mode 100644 index 42129944a19..00000000000 --- a/plugins/host-allocators/random/src/main/java/com/cloud/agent/manager/allocator/impl/RandomAllocator.java +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import javax.inject.Inject; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.ListUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; - -import com.cloud.agent.manager.allocator.HostAllocator; -import com.cloud.capacity.CapacityManager; -import com.cloud.dc.ClusterDetailsDao; -import com.cloud.dc.dao.ClusterDao; -import com.cloud.deploy.DeploymentPlan; -import com.cloud.deploy.DeploymentPlanner.ExcludeList; -import com.cloud.host.Host; -import com.cloud.host.Host.Type; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.offering.ServiceOffering; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.VMTemplateVO; -import com.cloud.utils.Pair; -import com.cloud.utils.component.AdapterBase; -import com.cloud.vm.VirtualMachine; -import com.cloud.vm.VirtualMachineProfile; - -@Component -public class RandomAllocator extends AdapterBase implements HostAllocator { - @Inject - private HostDao _hostDao; - @Inject - private ResourceManager _resourceMgr; - @Inject - private ClusterDao clusterDao; - @Inject - private ClusterDetailsDao clusterDetailsDao; - @Inject - private CapacityManager capacityManager; - - protected List listHostsByTags(Host.Type type, long dcId, Long podId, Long clusterId, String offeringHostTag, String templateTag) { - List taggedHosts = new ArrayList<>(); - if (offeringHostTag != null) { - taggedHosts.addAll(_hostDao.listByHostTag(type, clusterId, podId, dcId, offeringHostTag)); - } - if (templateTag != null) { - List templateTaggedHosts = _hostDao.listByHostTag(type, clusterId, podId, dcId, templateTag); - if (taggedHosts.isEmpty()) { - taggedHosts = templateTaggedHosts; - } else { - taggedHosts.retainAll(templateTaggedHosts); - } - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found %d hosts %s with type: %s, zone ID: %d, pod ID: %d, cluster ID: %s, offering host tag(s): %s, template tag: %s", - taggedHosts.size(), - (taggedHosts.isEmpty() ? "" : String.format("(%s)", StringUtils.join(taggedHosts.stream().map(HostVO::toString).toArray(), ","))), - type.name(), dcId, podId, clusterId, offeringHostTag, templateTag)); - } - return taggedHosts; - } - private List findSuitableHosts(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - long dcId = plan.getDataCenterId(); - Long podId = plan.getPodId(); - Long clusterId = plan.getClusterId(); - ServiceOffering offering = vmProfile.getServiceOffering(); - List hostsCopy = null; - List suitableHosts = new ArrayList<>(); - - if (type == Host.Type.Storage) { - return suitableHosts; - } - String offeringHostTag = offering.getHostTag(); - - VMTemplateVO template = (VMTemplateVO)vmProfile.getTemplate(); - String templateTag = template.getTemplateTag(); - String hostTag = null; - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostTag = ObjectUtils.allNotNull(offeringHostTag, templateTag) ? - String.format("%s, %s", offeringHostTag, templateTag) : - ObjectUtils.firstNonNull(offeringHostTag, templateTag); - logger.debug("Looking for hosts in dc [{}], pod [{}], cluster [{}] and complying with host tag(s): [{}]", dcId, podId, clusterId, hostTag); - } else { - logger.debug("Looking for hosts in dc: {} pod: {} cluster: {}", dcId , podId, clusterId); - } - if (hosts != null) { - // retain all computing hosts, regardless of whether they support routing...it's random after all - hostsCopy = new ArrayList<>(hosts); - if (ObjectUtils.anyNotNull(offeringHostTag, templateTag)) { - hostsCopy.retainAll(listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag)); - } else { - hostsCopy.retainAll(_hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId)); - } - } else { - // list all computing hosts, regardless of whether they support routing...it's random after all - if (offeringHostTag != null) { - hostsCopy = listHostsByTags(type, dcId, podId, clusterId, offeringHostTag, templateTag); - } else { - hostsCopy = _hostDao.listAllHostsThatHaveNoRuleTag(type, clusterId, podId, dcId); - } - } - hostsCopy = ListUtils.union(hostsCopy, _hostDao.findHostsWithTagRuleThatMatchComputeOferringTags(offeringHostTag)); - - if (hostsCopy.isEmpty()) { - logger.info("No suitable host found for VM [{}] in {}.", vmProfile, hostTag); - return null; - } - - logger.debug("Random Allocator found {} hosts", hostsCopy.size()); - if (hostsCopy.isEmpty()) { - return suitableHosts; - } - - Collections.shuffle(hostsCopy); - for (Host host : hostsCopy) { - if (suitableHosts.size() == returnUpTo) { - break; - } - if (avoid.shouldAvoid(host)) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Host %s is in avoid set, skipping this and trying other available hosts", host)); - } - continue; - } - Pair cpuCapabilityAndCapacity = capacityManager.checkIfHostHasCpuCapabilityAndCapacity(host, offering, considerReservedCapacity); - if (!cpuCapabilityAndCapacity.first() || !cpuCapabilityAndCapacity.second()) { - if (logger.isDebugEnabled()) { - logger.debug(String.format("Not using host %s; host has cpu capability? %s, host has capacity? %s", host, cpuCapabilityAndCapacity.first(), cpuCapabilityAndCapacity.second())); - } - continue; - } - if (logger.isDebugEnabled()) { - logger.debug(String.format("Found a suitable host, adding to list: %s", host)); - } - suitableHosts.add(host); - } - if (logger.isDebugEnabled()) { - logger.debug("Random Host Allocator returning " + suitableHosts.size() + " suitable hosts"); - } - return suitableHosts; - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, ExcludeList avoid, int returnUpTo) { - return allocateTo(vmProfile, plan, type, avoid, returnUpTo, true); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, Type type, - ExcludeList avoid, List hosts, int returnUpTo, - boolean considerReservedCapacity) { - if (CollectionUtils.isEmpty(hosts)) { - if (logger.isDebugEnabled()) { - logger.debug("Random Allocator found 0 hosts as given host list is empty"); - } - return new ArrayList<>(); - } - return findSuitableHosts(vmProfile, plan, type, avoid, hosts, returnUpTo, considerReservedCapacity); - } - - @Override - public List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan plan, - Type type, ExcludeList avoid, int returnUpTo, boolean considerReservedCapacity) { - return findSuitableHosts(vmProfile, plan, type, avoid, null, returnUpTo, considerReservedCapacity); - } - - @Override - public boolean isVirtualMachineUpgradable(VirtualMachine vm, ServiceOffering offering) { - // currently we do no special checks to rule out a VM being upgradable to an offering, so - // return true - return true; - } -} diff --git a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java b/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java deleted file mode 100644 index 538d7157184..00000000000 --- a/plugins/host-allocators/random/src/test/java/com/cloud/agent/manager/allocator/impl/RandomAllocatorTest.java +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.agent.manager.allocator.impl; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.collections.CollectionUtils; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; - -import com.cloud.host.Host; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; - -@RunWith(MockitoJUnitRunner.class) -public class RandomAllocatorTest { - - @Mock - HostDao hostDao; - @InjectMocks - RandomAllocator randomAllocator; - - @Test - public void testListHostsByTags() { - Host.Type type = Host.Type.Routing; - Long id = 1L; - String templateTag = "tag1"; - String offeringTag = "tag2"; - HostVO host1 = Mockito.mock(HostVO.class); - HostVO host2 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, offeringTag)).thenReturn(List.of(host1, host2)); - - // No template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(new ArrayList<>()); - List result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Different template tagged host - HostVO host3 = Mockito.mock(HostVO.class); - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host3)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertTrue(CollectionUtils.isEmpty(result)); - - // Matching template tagged host - Mockito.when(hostDao.listByHostTag(type, id, id, id, templateTag)).thenReturn(List.of(host1)); - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - - // No template tag - result = randomAllocator.listHostsByTags(type, id, id, id, offeringTag, null); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(2, result.size()); - - // No offering tag - result = randomAllocator.listHostsByTags(type, id, id, id, null, templateTag); - Assert.assertFalse(CollectionUtils.isEmpty(result)); - Assert.assertEquals(1, result.size()); - } -} diff --git a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java index 940897de3c9..c6c38a39809 100644 --- a/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java +++ b/plugins/hypervisors/baremetal/src/main/java/com/cloud/baremetal/manager/BareMetalTemplateAdapter.java @@ -106,7 +106,6 @@ public class BareMetalTemplateAdapter extends TemplateAdapterBase implements Tem } } - _resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template); return template; } diff --git a/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java b/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java index 3d79b9efdd1..4e44d8cb735 100644 --- a/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java +++ b/plugins/hypervisors/hyperv/src/main/java/com/cloud/ha/HypervInvestigator.java @@ -41,15 +41,15 @@ public class HypervInvestigator extends AdapterBase implements Investigator { @Override public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM { - Status status = isAgentAlive(host); + Status status = getHostAgentStatus(host); if (status == null) { throw new UnknownVM(); } - return status == Status.Up ? true : null; + return status == Status.Up; } @Override - public Status isAgentAlive(Host agent) { + public Status getHostAgentStatus(Host agent) { if (agent.getHypervisorType() != Hypervisor.HypervisorType.Hyperv) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java index ce9fbe6c232..da9a0d6e291 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/ha/KVMInvestigator.java @@ -19,10 +19,7 @@ package com.cloud.ha; import com.cloud.agent.AgentManager; -import com.cloud.agent.api.Answer; -import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.host.Host; -import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; @@ -34,11 +31,12 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreDriver; import org.apache.cloudstack.ha.HAManager; +import org.apache.cloudstack.kvm.ha.KVMHostActivityChecker; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import javax.inject.Inject; -import java.util.Arrays; +import java.util.Collections; import java.util.List; public class KVMInvestigator extends AdapterBase implements Investigator { @@ -54,13 +52,15 @@ public class KVMInvestigator extends AdapterBase implements Investigator { private HAManager haManager; @Inject private DataStoreProviderManager dataStoreProviderMgr; + @Inject + private KVMHostActivityChecker hostActivityChecker; @Override public boolean isVmAlive(com.cloud.vm.VirtualMachine vm, Host host) throws UnknownVM { if (haManager.isHAEligible(host)) { return haManager.isVMAliveOnHost(host); } - Status status = isAgentAlive(host); + Status status = getHostAgentStatus(host); logger.debug("HA: HOST is ineligible legacy state {} for host {}", status, host); if (status == null) { throw new UnknownVM(); @@ -73,86 +73,41 @@ public class KVMInvestigator extends AdapterBase implements Investigator { } @Override - public Status isAgentAlive(Host agent) { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + public Status getHostAgentStatus(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { return null; } - if (haManager.isHAEligible(agent)) { - return haManager.getHostStatus(agent); + if (haManager.isHAEligible(host)) { + return haManager.getHostStatusFromHAConfig(host); } - List clusterPools = _storagePoolDao.findPoolsInClusters(Arrays.asList(agent.getClusterId()), null); - boolean storageSupportHA = storageSupportHa(clusterPools); - if (!storageSupportHA) { - List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(agent.getDataCenterId(), agent.getHypervisorType()); - storageSupportHA = storageSupportHa(zonePools); + List clusterPools = _storagePoolDao.findPoolsInClusters(Collections.singletonList(host.getClusterId()), null); + boolean storageSupportsHA = storageSupportsHA(clusterPools); + if (!storageSupportsHA) { + List zonePools = _storagePoolDao.findZoneWideStoragePoolsByHypervisor(host.getDataCenterId(), host.getHypervisorType()); + storageSupportsHA = storageSupportsHA(zonePools); } - if (!storageSupportHA) { - logger.warn("Agent investigation was requested on host {}, but host does not support investigation because it has no NFS storage. Skipping investigation.", agent); + if (!storageSupportsHA) { + logger.warn("Agent investigation was requested on host {}, but host does not support investigation" + + " because it has no HA supported storage. Skipping investigation.", host); return null; } - Status hostStatus = null; - Status neighbourStatus = null; - boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); - CheckOnHostCommand cmd = new CheckOnHostCommand(agent, reportFailureIfOneStorageIsDown); - - try { - Answer answer = _agentMgr.easySend(agent.getId(), cmd); - if (answer != null) { - hostStatus = answer.getResult() ? Status.Down : Status.Up; - } - } catch (Exception e) { - logger.debug("Failed to send command to host: {}", agent); - } - if (hostStatus == null) { - hostStatus = Status.Disconnected; - } - - List neighbors = _resourceMgr.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); - for (HostVO neighbor : neighbors) { - if (neighbor.getId() == agent.getId() - || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { - continue; - } - logger.debug("Investigating host:{} via neighbouring host:{}", agent, neighbor); - try { - Answer answer = _agentMgr.easySend(neighbor.getId(), cmd); - if (answer != null) { - neighbourStatus = answer.getResult() ? Status.Down : Status.Up; - logger.debug("Neighbouring host:{} returned status:{} for the investigated host:{}", neighbor, neighbourStatus, agent); - if (neighbourStatus == Status.Up) { - break; - } - } - } catch (Exception e) { - logger.debug("Failed to send command to host: {}", neighbor); - } - } - if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Disconnected; - } - if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Down; - } - logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, agent); - return hostStatus; + return hostActivityChecker.getHostAgentStatus(host); } - private boolean storageSupportHa(List pools) { - boolean storageSupportHA = false; + private boolean storageSupportsHA(List pools) { for (StoragePoolVO pool : pools) { DataStoreProvider storeProvider = dataStoreProviderMgr.getDataStoreProvider(pool.getStorageProviderName()); DataStoreDriver storeDriver = storeProvider.getDataStoreDriver(); if (storeDriver instanceof PrimaryDataStoreDriver) { PrimaryDataStoreDriver primaryStoreDriver = (PrimaryDataStoreDriver)storeDriver; if (primaryStoreDriver.isStorageSupportHA(pool.getPoolType())) { - storageSupportHA = true; - break; + return true; } } } - return storageSupportHA; + return false; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java index 896426addca..f30e2c77919 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHABase.java @@ -35,11 +35,9 @@ import com.cloud.agent.properties.AgentPropertiesFileHandler; public class KVMHABase { protected Logger logger = LogManager.getLogger(getClass()); private long _timeout = 60000; /* 1 minutes */ - protected static String s_heartBeatPath; - protected long _heartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); - protected long _heartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + protected long _heartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); protected long _heartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); - protected long _heartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); + protected long _heartBeatUpdateRetrySleepInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); public static enum PoolType { PrimaryStorage, SecondaryStorage @@ -139,7 +137,7 @@ public class KVMHABase { /* Can't find the mount point? */ /* we need to mount it under poolName */ if (poolName != null) { - Script mount = new Script("/bin/bash", 60000); + Script mount = new Script("/bin/bash", _timeout); mount.add("-c"); mount.add("mount " + mountSource + " " + destPath); String result = mount.execute(); @@ -155,7 +153,6 @@ public class KVMHABase { } protected String getMountPoint(HAStoragePool storagePool) { - StoragePool pool = null; String poolName = null; try { @@ -172,7 +169,6 @@ public class KVMHABase { } poolName = pool.getName(); } - } catch (LibvirtException e) { logger.debug("Ignoring libvirt error.", e); } finally { @@ -235,7 +231,7 @@ public class KVMHABase { return result; } - public Boolean checkingHeartBeat() { + public Boolean hasHeartBeat() { // TODO Auto-generated method stub return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java index db6190fa8f2..0ee59c95da3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAChecker.java @@ -26,44 +26,43 @@ import com.cloud.agent.api.to.HostTO; public class KVMHAChecker extends KVMHABase implements Callable { private List storagePools; private HostTO host; - private boolean reportFailureIfOneStorageIsDown; + private boolean reportIfHeartBeatFailedForOneStoragePool; - public KVMHAChecker(List pools, HostTO host, boolean reportFailureIfOneStorageIsDown) { + public KVMHAChecker(List pools, HostTO host, boolean reportIfHeartBeatFailedForOneStoragePool) { this.storagePools = pools; this.host = host; - this.reportFailureIfOneStorageIsDown = reportFailureIfOneStorageIsDown; + this.reportIfHeartBeatFailedForOneStoragePool = reportIfHeartBeatFailedForOneStoragePool; } /* - * True means heartbeaing is on going, or we can't get it's status. False - * means heartbeating is stopped definitely + * True means heart beating is on going, or we can't get it's status. + * False means heart beating is stopped definitely. */ @Override - public Boolean checkingHeartBeat() { - boolean validResult = false; - - String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); - - logger.debug(String.format("Checking heart beat with KVMHAChecker for %s", hostAndPools)); + public Boolean hasHeartBeat() { + String hostAndPools = String.format("host IP [%s] in pools [%s]", host.getPrivateNetwork().getIp(), + storagePools.stream().map(pool -> pool.getPoolUUID()).collect(Collectors.joining(", "))); + logger.debug("Checking heart beat with KVMHAChecker for {}", hostAndPools); + boolean heartBeatCheckResult = false; for (HAStoragePool pool : storagePools) { - validResult = pool.getPool().checkingHeartBeat(pool, host); - if (reportFailureIfOneStorageIsDown && !validResult) { + heartBeatCheckResult = pool.getPool().hasHeartBeat(pool, host); + if (reportIfHeartBeatFailedForOneStoragePool && !heartBeatCheckResult) { break; } } - if (!validResult) { - logger.warn(String.format("All checks with KVMHAChecker for %s considered it as dead. It may cause a shutdown of the host.", hostAndPools)); + if (!heartBeatCheckResult) { + logger.warn("All checks with KVMHAChecker for {} considered it as dead. It may cause a shutdown of the host.", hostAndPools); } - return validResult; + return heartBeatCheckResult; } @Override public Boolean call() throws Exception { // logger.addAppender(new org.apache.log4j.ConsoleAppender(new // org.apache.log4j.PatternLayout(), "System.out")); - return checkingHeartBeat(); + return hasHeartBeat(); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java index cf407bfc08a..9f1b849e972 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAMonitor.java @@ -18,7 +18,7 @@ package com.cloud.hypervisor.kvm.resource; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; -import com.cloud.storage.Storage.StoragePoolType; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.utils.script.Script; import org.libvirt.Connect; import org.libvirt.LibvirtException; @@ -34,60 +34,51 @@ import java.util.concurrent.ConcurrentHashMap; public class KVMHAMonitor extends KVMHABase implements Runnable { - private final Map storagePool = new ConcurrentHashMap<>(); + private final Map haStoragePools = new ConcurrentHashMap<>(); private final boolean rebootHostAndAlertManagementOnHeartbeatTimeout; private final String hostPrivateIp; - public KVMHAMonitor(HAStoragePool pool, String host, String scriptPath) { - if (pool != null) { - storagePool.put(pool.getPoolUUID(), pool); - } + public KVMHAMonitor(String host) { hostPrivateIp = host; - configureHeartBeatPath(scriptPath); - rebootHostAndAlertManagementOnHeartbeatTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT); } - private static synchronized void configureHeartBeatPath(String scriptPath) { - KVMHABase.s_heartBeatPath = scriptPath; - } - public void addStoragePool(HAStoragePool pool) { - synchronized (storagePool) { - storagePool.put(pool.getPoolUUID(), pool); + synchronized (haStoragePools) { + haStoragePools.put(pool.getPoolUUID(), pool); } } public void removeStoragePool(String uuid) { - synchronized (storagePool) { - HAStoragePool pool = storagePool.get(uuid); + synchronized (haStoragePools) { + HAStoragePool pool = haStoragePools.get(uuid); if (pool != null) { Script.runSimpleBashScript("umount " + pool.getMountDestPath()); - storagePool.remove(uuid); + haStoragePools.remove(uuid); } } } public List getStoragePools() { - synchronized (storagePool) { - return new ArrayList<>(storagePool.values()); + synchronized (haStoragePools) { + return new ArrayList<>(haStoragePools.values()); } } public HAStoragePool getStoragePool(String uuid) { - synchronized (storagePool) { - return storagePool.get(uuid); + synchronized (haStoragePools) { + return haStoragePools.get(uuid); } } protected void runHeartBeat() { - synchronized (storagePool) { + synchronized (haStoragePools) { Set removedPools = new HashSet<>(); - for (String uuid : storagePool.keySet()) { - HAStoragePool primaryStoragePool = storagePool.get(uuid); - if (primaryStoragePool.getPool().getType() == StoragePoolType.NetworkFilesystem) { - checkForNotExistingPools(removedPools, uuid); + for (String uuid : haStoragePools.keySet()) { + HAStoragePool primaryStoragePool = haStoragePools.get(uuid); + if (HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(primaryStoragePool.getPool().getType())) { + checkForNotExistingLibvirtStoragePools(removedPools, uuid); if (removedPools.contains(uuid)) { continue; } @@ -96,7 +87,7 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { result = executePoolHeartBeatCommand(uuid, primaryStoragePool, result); if (result != null && rebootHostAndAlertManagementOnHeartbeatTimeout) { - logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; stopping cloudstack-agent.", uuid, result)); + logger.warn("Write heartbeat for pool [{}] failed: {}; stopping cloudstack-agent.", uuid, result); primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, null, false);; } } @@ -109,45 +100,43 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { } private String executePoolHeartBeatCommand(String uuid, HAStoragePool primaryStoragePool, String result) { - for (int i = 1; i <= _heartBeatUpdateMaxTries; i++) { + for (int attempt = 1; attempt <= _heartBeatUpdateMaxTries; attempt++) { result = primaryStoragePool.getPool().createHeartBeatCommand(primaryStoragePool, hostPrivateIp, true); - - if (result != null) { - logger.warn(String.format("Write heartbeat for pool [%s] failed: %s; try: %s of %s.", uuid, result, i, _heartBeatUpdateMaxTries)); - try { - Thread.sleep(_heartBeatUpdateRetrySleep); - } catch (InterruptedException e) { - logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); - } - } else { + if (result == null) { break; } + logger.warn("Write heartbeat for pool [{}] failed: {}; try: {} of {}.", uuid, result, attempt, _heartBeatUpdateMaxTries); + try { + Thread.sleep(_heartBeatUpdateRetrySleepInMs); + } catch (InterruptedException e) { + logger.debug("[IGNORED] Interrupted between heartbeat retries.", e); + } } return result; } - private void checkForNotExistingPools(Set removedPools, String uuid) { + private void checkForNotExistingLibvirtStoragePools(Set removedPools, String uuid) { try { Connect conn = LibvirtConnection.getConnection(); StoragePool storage = conn.storagePoolLookupByUUIDString(uuid); if (storage == null || storage.getInfo().state != StoragePoolState.VIR_STORAGE_POOL_RUNNING) { if (storage == null) { - logger.debug(String.format("Libvirt storage pool [%s] not found, removing from HA list.", uuid)); + logger.debug("Libvirt storage pool [{}] not found, removing from HA list.", uuid); } else { - logger.debug(String.format("Libvirt storage pool [%s] found, but not running, removing from HA list.", uuid)); + logger.debug("Libvirt storage pool [{}] found, but not running, removing from HA list.", uuid); } removedPools.add(uuid); } - logger.debug(String.format("Found NFS storage pool [%s] in libvirt, continuing.", uuid)); + logger.debug("Found NFS storage pool [{}] in libvirt, continuing.", uuid); } catch (LibvirtException e) { - logger.debug(String.format("Failed to lookup libvirt storage pool [%s].", uuid), e); + logger.debug("Failed to lookup libvirt storage pool [{}].", uuid, e); if (e.toString().contains("pool not found")) { - logger.debug(String.format("Removing pool [%s] from HA monitor since it was deleted.", uuid)); + logger.debug("Removing pool [{}] from HA monitor since it was deleted.", uuid); removedPools.add(uuid); } } @@ -160,11 +149,10 @@ public class KVMHAMonitor extends KVMHABase implements Runnable { runHeartBeat(); try { - Thread.sleep(_heartBeatUpdateFreq); + Thread.sleep(_heartBeatUpdateFreqInMs); } catch (InterruptedException e) { logger.debug("[IGNORED] Interrupted between heartbeats.", e); } } } - } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java index e6937b515e9..c13be64a3a3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/KVMHAVMActivityChecker.java @@ -39,12 +39,12 @@ public class KVMHAVMActivityChecker extends KVMHABase implements Callable convertInstanceEnv = null; + private String vddkLibDir = null; + private static final String libguestfsBackend = "direct"; protected boolean dpdkSupport = false; protected String dpdkOvsPath; protected String directDownloadTemporaryDownloadPath; protected String cachePath; + private String vddkTransports = null; + private String vddkThumbprint = null; + private String vddkVersion = null; + private String detectedPasswordFileOption = null; protected String javaTempDir = System.getProperty("java.io.tmpdir"); private String getEndIpFromStartIp(final String startIp, final int numIps) { @@ -953,6 +983,26 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return convertInstanceEnv; } + public String getVddkLibDir() { + return vddkLibDir; + } + + public String getLibguestfsBackend() { + return libguestfsBackend; + } + + public String getVddkTransports() { + return vddkTransports; + } + + public String getVddkThumbprint() { + return vddkThumbprint; + } + + public String getVddkVersion() { + return vddkVersion; + } + /** * Defines resource's public and private network interface according to what is configured in agent.properties. */ @@ -1065,11 +1115,6 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv throw new ConfigurationException("Unable to find patch.sh"); } - heartBeatPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (heartBeatPath == null) { - throw new ConfigurationException("Unable to find kvmheartbeat.sh"); - } - createVmPath = Script.findScript(storageScriptsDir, "createvm.sh"); if (createVmPath == null) { throw new ConfigurationException("Unable to find the createvm.sh"); @@ -1158,6 +1203,37 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv setConvertInstanceEnv(convertEnvTmpDir, convertEnvVirtv2vTmpDir); + vddkLibDir = StringUtils.trimToNull(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_LIB_DIR)); + if (StringUtils.isNotBlank(vddkLibDir) && !isVddkLibDirValid(vddkLibDir)) { + LOGGER.warn("Configured VDDK library dir [{}] is invalid (missing lib64/libvixDiskLib.so), attempting auto-detection", vddkLibDir); + vddkLibDir = null; + } + if (StringUtils.isBlank(vddkLibDir)) { + vddkLibDir = detectVddkLibDir(); + } + if (StringUtils.isNotBlank(vddkLibDir)) { + LOGGER.info("Detected VDDK library dir: {}", vddkLibDir); + } else { + LOGGER.warn("Could not detect a valid VDDK library dir; VDDK conversion will be unavailable"); + } + + vddkVersion = detectVddkVersion(); + if (StringUtils.isNotBlank(vddkVersion)) { + LOGGER.info("Detected nbdkit VDDK plugin version: {}", vddkVersion); + } + + vddkTransports = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_TRANSPORTS)); + vddkThumbprint = StringUtils.trimToNull( + AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VDDK_THUMBPRINT)); + + detectedPasswordFileOption = detectPasswordFileOption(); + if (StringUtils.isNotBlank(detectedPasswordFileOption)) { + LOGGER.info("Detected virt-v2v password option: {}", detectedPasswordFileOption); + } else { + LOGGER.warn("Could not detect virt-v2v password option, VDDK conversions may fail"); + } + pool = (String)params.get("pool"); if (pool == null) { pool = "/root"; @@ -1332,9 +1408,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv final String[] info = NetUtils.getNetworkParams(privateNic); - kvmhaMonitor = new KVMHAMonitor(null, info[0], heartBeatPath); - final Thread ha = new Thread(kvmhaMonitor); - ha.start(); + kvmhaMonitor = new KVMHAMonitor(info[0]); + final Thread haMonitorThread = new Thread(kvmhaMonitor); + haMonitorThread.start(); storagePoolManager = new KVMStoragePoolManager(storageLayer, kvmhaMonitor); @@ -2909,25 +2985,63 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv protected void setQuotaAndPeriod(VirtualMachineTO vmTO, CpuTuneDef ctd) { if (vmTO.isLimitCpuUse() && vmTO.getCpuQuotaPercentage() != null) { Double cpuQuotaPercentage = vmTO.getCpuQuotaPercentage(); - int period = CpuTuneDef.DEFAULT_PERIOD; - int quota = (int) (period * cpuQuotaPercentage); - if (quota < CpuTuneDef.MIN_QUOTA) { - LOGGER.info("Calculated quota (" + quota + ") below the minimum (" + CpuTuneDef.MIN_QUOTA + ") for VM domain " + vmTO.getUuid() + ", setting it to minimum " + - "and calculating period instead of using the default"); - quota = CpuTuneDef.MIN_QUOTA; - period = (int) ((double) quota / cpuQuotaPercentage); - if (period > CpuTuneDef.MAX_PERIOD) { - LOGGER.info("Calculated period (" + period + ") exceeds the maximum (" + CpuTuneDef.MAX_PERIOD + - "), setting it to the maximum"); - period = CpuTuneDef.MAX_PERIOD; - } - } - ctd.setQuota(quota); - ctd.setPeriod(period); - LOGGER.info("Setting quota=" + quota + ", period=" + period + " to VM domain " + vmTO.getUuid()); + Pair periodAndQuota = getPeriodAndQuota(cpuQuotaPercentage); + ctd.setPeriod(periodAndQuota.first()); + ctd.setQuota(periodAndQuota.second()); + LOGGER.info("Setting quota = [{}] and period = [{}] to VM domain [{}].", periodAndQuota.second(), periodAndQuota.first(), vmTO.getUuid()); } } + /** + * Calculates the CPU period and quota based on the quota percentage defined by the Management Server + * @param cpuQuotaPercentage CPU quota percentage defined by the Management Server + * @return The period and quota to be defined for the VM's domain + */ + protected Pair getPeriodAndQuota(double cpuQuotaPercentage) { + int period = CpuTuneDef.DEFAULT_PERIOD; + long quota = (long) (period * cpuQuotaPercentage); + if (quota < CpuTuneDef.MIN_QUOTA) { + LOGGER.info("Calculated quota ({}) below the minimum ({}), setting it to minimum and calculating period instead of using the default", quota, CpuTuneDef.MIN_QUOTA); + quota = CpuTuneDef.MIN_QUOTA; + period = (int) ((double) quota / cpuQuotaPercentage); + if (period > CpuTuneDef.MAX_PERIOD) { + LOGGER.info("Calculated period ({}) exceeds the maximum ({}), setting it to the maximum", period, CpuTuneDef.MAX_PERIOD); + period = CpuTuneDef.MAX_PERIOD; + } + } + + LOGGER.info("Calculated period = [{}] and quota = [{}] given the [{}] quota percentage.", period, quota, cpuQuotaPercentage); + return new Pair<>(period, quota); + } + + /** + * Dynamically updates the domain's "vcpu_quota" and "period" fields of the CPU tune definition. + * This is required because the values of the fields must change according to the new CPU speed of the VM. + * When the CPU limitation is removed from the domain, the "vcpu_quota" field is set to 17,592,186,044,415. + * @param domain VM's domain. + * @param vmTO VM's transfer object, which contains the required fields to update the "vcpu_quota" and "period" fields. + * @param limitCpuUseChange Indicates whether the CPU limitation for the VM has changed. + * @throws org.libvirt.LibvirtException + **/ + public void updateCpuQuotaAndPeriod(Domain domain, VirtualMachineTO vmTO, boolean limitCpuUseChange) throws LibvirtException { + if (hypervisorLibvirtVersion < MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE || (!limitCpuUseChange && !vmTO.isLimitCpuUse())) { + logger.info("Not updating the [{}] and [{}] for the [{}] domain, because [{}].", + CpuSchedulerParameter.QUOTA, CpuSchedulerParameter.PERIOD, domain.getName(), hypervisorLibvirtVersion < MIN_LIBVIRT_VERSION_FOR_GUEST_CPU_TUNE ? + "the current Libvirt version does not support CPU tune" : "it was not requested to remove, change or apply CPU limitation for the instance."); + return; + } + + if (limitCpuUseChange && !vmTO.isLimitCpuUse()) { + logger.info("Updating the [{}] of the [{}] domain to [{}], because CPU limitation has been removed.", CpuSchedulerParameter.QUOTA, domain.getName(), CpuTuneDef.MAX_CPU_QUOTA); + LibvirtComputingResource.setQuota(domain, CpuTuneDef.MAX_CPU_QUOTA); + return; + } + + Pair periodAndQuota = getPeriodAndQuota(vmTO.getCpuQuotaPercentage()); + LibvirtComputingResource.setPeriod(domain, periodAndQuota.first()); + LibvirtComputingResource.setQuota(domain, periodAndQuota.second()); + } + protected void enlightenWindowsVm(VirtualMachineTO vmTO, FeaturesDef features) { if (vmTO.getOs().contains("Windows PV")) { // If OS is Windows PV, then enable the features. Features supported on Windows 2008 and later @@ -3389,10 +3503,10 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv grd.setMemBalloning(!noMemBalloon); - Long maxRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMaxRam()); - - grd.setMemorySize(maxRam); - grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, maxRam)); + long requestedRam = ByteScaleUtils.bytesToKibibytes(vmTO.getRequestedRam()); + long minRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMinRam()); + grd.setCurrentMem(getCurrentMemAccordingToMemBallooning(vmTO, requestedRam, minRam)); + grd.setMaxMemory(ByteScaleUtils.bytesToKibibytes(vmTO.getMaxRam())); int vcpus = vmTO.getCpus(); Integer maxVcpus = vmTO.getVcpuMaxLimit(); @@ -3403,18 +3517,19 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return grd; } - protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long maxRam) { - long retVal = maxRam; + protected long getCurrentMemAccordingToMemBallooning(VirtualMachineTO vmTO, long requestedRam, long minRam) { if (noMemBalloon) { - LOGGER.warn(String.format("Setting VM's [%s] current memory as max memory [%s] due to memory ballooning is disabled. If you are using a custom service offering, verify if memory ballooning really should be disabled.", vmTO.toString(), maxRam)); - } else if (vmTO != null && vmTO.getType() != VirtualMachine.Type.User) { - LOGGER.warn(String.format("Setting System VM's [%s] current memory as max memory [%s].", vmTO.toString(), maxRam)); - } else { - long minRam = ByteScaleUtils.bytesToKibibytes(vmTO.getMinRam()); - LOGGER.debug(String.format("Setting VM's [%s] current memory as min memory [%s] due to memory ballooning is enabled.", vmTO.toString(), minRam)); - retVal = minRam; + LOGGER.warn("Setting VM's [{}] current memory as requested memory [{}] due to memory ballooning is disabled.", vmTO.toString(), requestedRam); + return requestedRam; } - return retVal; + + if (vmTO != null && vmTO.getType() != VirtualMachine.Type.User) { + LOGGER.warn("Setting System VM's [{}] current memory as requested memory [{}].", vmTO.toString(), requestedRam); + return requestedRam; + } + + LOGGER.debug("Setting VM's [{}] current memory as min memory [{}] due to memory ballooning is enabled.", vmTO.toString(), minRam); + return minRam; } /** @@ -4229,6 +4344,13 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv cmd.setHostTags(getHostTags()); boolean instanceConversionSupported = hostSupportsInstanceConversion(); cmd.getHostDetails().put(HOST_INSTANCE_CONVERSION, String.valueOf(instanceConversionSupported)); + cmd.getHostDetails().put(HOST_VDDK_SUPPORT, String.valueOf(hostSupportsVddk())); + if (StringUtils.isNotBlank(vddkLibDir)) { + cmd.getHostDetails().put(HOST_VDDK_LIB_DIR, vddkLibDir); + } + if (StringUtils.isNotBlank(vddkVersion)) { + cmd.getHostDetails().put(HOST_VDDK_VERSION, vddkVersion); + } if (instanceConversionSupported) { cmd.getHostDetails().put(HOST_VIRTV2V_VERSION, getHostVirtV2vVersion()); } @@ -5950,6 +6072,66 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + public boolean hostSupportsVddk() { + return hostSupportsVddk(null); + } + + public boolean hostSupportsVddk(String overriddenVddkLibDir) { + String effectiveVddkLibDir = StringUtils.trimToNull(overriddenVddkLibDir); + if (StringUtils.isBlank(effectiveVddkLibDir)) { + effectiveVddkLibDir = StringUtils.trimToNull(vddkLibDir); + } + if (StringUtils.isBlank(effectiveVddkLibDir) || !isVddkLibDirValid(effectiveVddkLibDir)) { + effectiveVddkLibDir = detectVddkLibDir(); + } + return hostSupportsInstanceConversion() && isVddkLibDirValid(effectiveVddkLibDir) && StringUtils.isNotBlank(detectVddkVersion()); + } + + protected boolean isVddkLibDirValid(String path) { + if (StringUtils.isBlank(path)) { + return false; + } + File libDir = new File(path, "lib64"); + if (!libDir.isDirectory()) { + return false; + } + File[] libs = libDir.listFiles((dir, name) -> name.startsWith("libvixDiskLib.so")); + return libs != null && libs.length > 0; + } + + protected String detectVddkLibDir() { + String detectedPath = StringUtils.trimToNull(Script.runSimpleBashScript(VDDK_AUTODETECT_PATH_CMD)); + if (StringUtils.isNotBlank(detectedPath) && isVddkLibDirValid(detectedPath)) { + return detectedPath; + } + return null; + } + + protected String detectVddkVersion() { + try { + ProcessBuilder pb = new ProcessBuilder("nbdkit", "vddk", "--version"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (StringUtils.isBlank(output)) { + return null; + } + + for (String line : output.split("\\R")) { + String trimmed = StringUtils.trimToEmpty(line); + if (trimmed.startsWith("vddk ")) { + return StringUtils.trimToNull(trimmed.substring("vddk ".length())); + } + } + return null; + } catch (Exception e) { + LOGGER.error("Failed to detect vddk version: {}", e.getMessage()); + return null; + } + } + public boolean hostSupportsWindowsGuestConversion() { if (isUbuntuOrDebianHost()) { int exitValue = Script.runSimpleBashScriptForExitValue(UBUNTU_WINDOWS_GUEST_CONVERSION_SUPPORTED_CHECK_CMD); @@ -5964,6 +6146,40 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return exitValue == 0; } + /** + * Detect which password option virt-v2v supports by examining its --help output + * @return "-ip" if supported (virt-v2v >= 2.8.1), "--password-file" if older version, or null if detection fails + */ + protected String detectPasswordFileOption() { + try { + ProcessBuilder pb = new ProcessBuilder("virt-v2v", "--help"); + Process process = pb.start(); + + String output = new String(process.getInputStream().readAllBytes()); + process.waitFor(); + + if (output.contains("-ip ")) { + return "-ip"; + } else if (output.contains("--password-file")) { + return "--password-file"; + } else { + LOGGER.error("virt-v2v does not support -ip or --password-file"); + return null; + } + } catch (Exception e) { + LOGGER.error("Failed to detect virt-v2v password option: {}", e.getMessage()); + return null; + } + } + + /** + * Get the detected password file option for virt-v2v + * @return the password option ("-ip" or "--password-file") or null if not detected + */ + public String getDetectedPasswordFileOption() { + return detectedPasswordFileOption; + } + public String getHostVirtV2vVersion() { if (!hostSupportsInstanceConversion()) { return ""; @@ -6098,29 +6314,59 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv **/ public static Integer getCpuShares(Domain dm) throws LibvirtException { for (SchedParameter c : dm.getSchedulerParameters()) { - if (c.field.equals("cpu_shares")) { + if (c.field.equals(CpuSchedulerParameter.CPU_SHARES.getName())) { return Integer.parseInt(c.getValueAsString()); } } - LOGGER.warn(String.format("Could not get cpu_shares of domain: [%s]. Returning default value of 0. ", dm.getName())); + LOGGER.warn("Could not get [{}] of domain: [{}]. Returning default value of 0. ", CpuSchedulerParameter.CPU_SHARES.getName(), dm.getName()); return 0; } /** - * Sets the cpu_shares (priority) of the running VM
+ * Updates the cpu_shares (priority) of the running VM. * @param dm domain of the VM. * @param cpuShares new priority of the running VM. - * @throws org.libvirt.LibvirtException **/ public static void setCpuShares(Domain dm, Integer cpuShares) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.CPU_SHARES.getName(), dm.getName(), cpuShares); SchedUlongParameter[] params = new SchedUlongParameter[1]; params[0] = new SchedUlongParameter(); - params[0].field = "cpu_shares"; + params[0].field = CpuSchedulerParameter.CPU_SHARES.getName(); params[0].value = cpuShares; dm.setSchedulerParameters(params); } + /** + * Updates the period of the running VM. + * @param domain domain of the VM. + * @param period new period of the running VM. + **/ + public static void setPeriod(Domain domain, int period) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.PERIOD.getName(), domain.getName(), period); + SchedUlongParameter[] params = new SchedUlongParameter[1]; + params[0] = new SchedUlongParameter(); + params[0].field = CpuSchedulerParameter.PERIOD.getName(); + params[0].value = period; + + domain.setSchedulerParameters(params); + } + + /** + * Updates the quota of the running VM. + * @param domain domain of the VM. + * @param quota new quota of the running VM. + **/ + public static void setQuota(Domain domain, long quota) throws LibvirtException { + LOGGER.info("Dynamically updating the [{}] of the [{}] VM to [{}].", CpuSchedulerParameter.QUOTA.getName(), domain.getName(), quota); + SchedLongParameter[] params = new SchedLongParameter[1]; + params[0] = new SchedLongParameter(); + params[0].field = CpuSchedulerParameter.QUOTA.getName(); + params[0].value = quota; + + domain.setSchedulerParameters(params); + } + /** * Set up a libvirt secret for a volume. If Libvirt says that a secret already exists for this volume path, we use its uuid. * The UUID of the secret needs to be prescriptive such that we can register the same UUID on target host during live migration diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java index 4d823783a99..e114669b8b5 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtDomainXMLParser.java @@ -542,7 +542,7 @@ public class LibvirtDomainXMLParser { final String quota = getTagValue("quota", cpuTuneDefElement); if (StringUtils.isNotBlank(quota)) { - cpuTuneDef.setQuota((Integer.parseInt(quota))); + cpuTuneDef.setQuota((Long.parseLong(quota))); } final String period = getTagValue("period", cpuTuneDefElement); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java index 08086859fb7..d3765c01ccb 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDef.java @@ -79,28 +79,47 @@ public class LibvirtGpuDef { gpuBuilder.append(" \n"); gpuBuilder.append(" \n"); - // Parse the bus address (e.g., 00:02.0) into domain, bus, slot, function - String domain = "0x0000"; - String bus = "0x00"; - String slot = "0x00"; - String function = "0x0"; + // Parse the bus address into domain, bus, slot, function. Two input formats are accepted: + // - "dddd:bb:ss.f" full PCI address with domain (e.g. 0000:00:02.0) + // - "bb:ss.f" legacy short BDF; domain defaults to 0000 + // Each segment is parsed as a hex integer and formatted with fixed widths + // (domain: 4 hex digits, bus/slot: 2 hex digits, function: 1 hex digit) to + // produce canonical libvirt XML values regardless of input casing or padding. + int domainVal = 0, busVal = 0, slotVal = 0, funcVal = 0; if (busAddress != null && !busAddress.isEmpty()) { String[] parts = busAddress.split(":"); - if (parts.length > 1) { - bus = "0x" + parts[0]; - String[] slotFunctionParts = parts[1].split("\\."); - if (slotFunctionParts.length > 0) { - slot = "0x" + slotFunctionParts[0]; - if (slotFunctionParts.length > 1) { - function = "0x" + slotFunctionParts[1].trim(); - } + try { + String slotFunction; + if (parts.length == 3) { + domainVal = Integer.parseInt(parts[0], 16); + busVal = Integer.parseInt(parts[1], 16); + slotFunction = parts[2]; + } else if (parts.length == 2) { + busVal = Integer.parseInt(parts[0], 16); + slotFunction = parts[1]; + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); } + String[] sf = slotFunction.split("\\."); + if (sf.length == 2) { + slotVal = Integer.parseInt(sf[0], 16); + funcVal = Integer.parseInt(sf[1].trim(), 16); + } else { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'"); + } + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid PCI bus address format: '" + busAddress + "'", e); } } + String domain = String.format("0x%04x", domainVal); + String bus = String.format("0x%02x", busVal); + String slot = String.format("0x%02x", slotVal); + String function = String.format("0x%x", funcVal); + gpuBuilder.append("

\n"); + .append(slot).append("' function='").append(function).append("'/>\n"); gpuBuilder.append(" \n"); gpuBuilder.append("\n"); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java index 097a9b8dd32..bf8b1af6c18 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java @@ -443,15 +443,15 @@ public class LibvirtVMDef { } public static class GuestResourceDef { - private long memory; + private long maxMemory; private long currentMemory = -1; private int vcpu = -1; private int maxVcpu = -1; private boolean memoryBalloning = false; private int memoryBalloonStatsPeriod = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_MEMBALLOON_STATS_PERIOD); - public void setMemorySize(long mem) { - this.memory = mem; + public void setMaxMemory(long mem) { + this.maxMemory = mem; } public void setCurrentMem(long currMem) { @@ -484,8 +484,8 @@ public class LibvirtVMDef { response.append(String.format("%s\n", this.currentMemory)); response.append(String.format("%s\n", this.currentMemory)); - if (this.memory > this.currentMemory) { - response.append(String.format("%s\n", this.memory)); + if (this.maxMemory > this.currentMemory) { + response.append(String.format("%s\n", this.maxMemory)); response.append(String.format(" \n", this.maxVcpu - 1, this.currentMemory)); } @@ -1920,11 +1920,12 @@ public class LibvirtVMDef { public static class CpuTuneDef { private int _shares = 0; - private int quota = 0; + private long quota = 0; private int period = 0; static final int DEFAULT_PERIOD = 10000; static final int MIN_QUOTA = 1000; static final int MAX_PERIOD = 1000000; + public static final long MAX_CPU_QUOTA = 17592186044415L; public void setShares(int shares) { _shares = shares; @@ -1934,11 +1935,11 @@ public class LibvirtVMDef { return _shares; } - public int getQuota() { + public long getQuota() { return quota; } - public void setQuota(int quota) { + public void setQuota(long quota) { this.quota = quota; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java index b94b4830003..de9341715f0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapper.java @@ -30,7 +30,15 @@ public class LibvirtCheckConvertInstanceCommandWrapper extends CommandWrapper pools = monitor.getStoragePools(); final HostTO host = command.getHost(); - final KVMHAChecker ha = new KVMHAChecker(pools, host, command.isCheckFailedOnOneStorage()); + + final KVMHAChecker ha = new KVMHAChecker(pools, host, command.shouldReportIfHeartBeatFailedForOneStoragePool()); final Future future = executors.submit(ha); try { - final Boolean result = future.get(); - if (result) { - return new Answer(command, false, "Heart is beating..."); + final Boolean hasHeartBeat = future.get(); + if (hasHeartBeat) { + return new CheckOnHostAnswer(command, true, "Heart is beating"); } else { - return new Answer(command); + return new CheckOnHostAnswer(command, "Heart is not beating"); } } catch (final InterruptedException e) { - return new Answer(command, false, "CheckOnHostCommand: can't get status of host: InterruptedException"); + return new CheckOnHostAnswer(command, "CheckOnHostCommand: can't get status of host: InterruptedException"); } catch (final ExecutionException e) { - return new Answer(command, false, "CheckOnHostCommand: can't get status of host: ExecutionException"); + return new CheckOnHostAnswer(command, "CheckOnHostCommand: can't get status of host: ExecutionException"); } } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java index a708d441be5..b132f6d98ce 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVMActivityOnStoragePoolCommandWrapper.java @@ -48,9 +48,9 @@ public final class LibvirtCheckVMActivityOnStoragePoolCommandWrapper extends Com KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(pool.getType(), pool.getUuid()); - if (primaryPool.isPoolSupportHA()){ - final HAStoragePool nfspool = monitor.getStoragePool(pool.getUuid()); - final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(nfspool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); + if (primaryPool.isPoolSupportHA()) { + final HAStoragePool haPool = monitor.getStoragePool(pool.getUuid()); + final KVMHAVMActivityChecker ha = new KVMHAVMActivityChecker(haPool, command.getHost(), command.getVolumeList(), libvirtComputingResource.getVmActivityCheckPath(), command.getSuspectTimeInSeconds()); final Future future = executors.submit(ha); try { final Boolean result = future.get(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java index 99005755cbb..aa6a37b6561 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapper.java @@ -45,11 +45,10 @@ public final class LibvirtCheckVirtualMachineCommandWrapper extends CommandWrapp Integer vncPort = null; if (state == PowerState.PowerOn) { vncPort = libvirtComputingResource.getVncPort(conn, command.getVmName()); - } - - Domain vm = conn.domainLookupByName(command.getVmName()); - if (state == PowerState.PowerOn && DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { - return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + Domain vm = conn.domainLookupByName(command.getVmName()); + if (DomainInfo.DomainState.VIR_DOMAIN_PAUSED.equals(vm.getInfo().state)) { + return new CheckVirtualMachineAnswer(command, PowerState.PowerUnknown, vncPort); + } } return new CheckVirtualMachineAnswer(command, state, vncPort); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 55225ba086c..6788516df74 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -47,7 +47,10 @@ import java.util.Map; @ResourceWrapper(handles = CheckVolumeCommand.class) public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper { - private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem); + private static final List STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( + Storage.StoragePoolType.Filesystem, + Storage.StoragePoolType.NetworkFilesystem, + Storage.StoragePoolType.SharedMountPoint); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java index 66a5f5dd7d2..f28b5eda4a6 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapper.java @@ -20,10 +20,17 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.net.URLEncoder; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Locale; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.commons.collections4.MapUtils; @@ -51,6 +58,7 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper supportedInstanceConvertSourceHypervisors = List.of(Hypervisor.HypervisorType.VMware); + private static final Pattern SHA1_FINGERPRINT_PATTERN = Pattern.compile("(?i)(?:SHA1\\s+)?Fingerprint\\s*=\\s*([0-9A-F:]+)"); @Override public Answer execute(ConvertInstanceCommand cmd, LibvirtComputingResource serverResource) { @@ -61,7 +69,8 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { - noOfThreads = 0; - } - ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); - temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); - if (!ovfExported) { - String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); - logger.error(String.format("(%s) %s", originalVMName, err)); - return new Answer(cmd, false, err); - } - sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); - } else { - ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); - sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); - } - - logger.info(String.format("(%s) Attempting to convert the OVF %s of the instance %s from %s to KVM", - originalVMName, ovfTemplateDirOnConversionLocation, sourceInstanceName, sourceHypervisorType)); - final String temporaryConvertUuid = UUID.randomUUID().toString(); boolean verboseModeEnabled = serverResource.isConvertInstanceVerboseModeEnabled(); boolean cleanupSecondaryStorage = false; + boolean ovfExported = false; + String ovfTemplateDirOnConversionLocation = null; + try { - boolean result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, - timeout, verboseModeEnabled, extraParams, serverResource); + boolean result; + if (useVddk) { + logger.info("({}) Using VDDK-based conversion (direct from VMware)", originalVMName); + String vddkLibDir = resolveVddkSetting(cmd.getVddkLibDir(), serverResource.getVddkLibDir()); + if (StringUtils.isBlank(vddkLibDir)) { + String err = String.format("VDDK lib dir is not configured on the host. " + + "Set '%s' in agent.properties or in details parameter of the import api call to use VDDK-based conversion.", "vddk.lib.dir"); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + String vddkTransports = resolveVddkSetting(cmd.getVddkTransports(), serverResource.getVddkTransports()); + String configuredVddkThumbprint = resolveVddkSetting(cmd.getVddkThumbprint(), serverResource.getVddkThumbprint()); + String passwordOption = serverResource.getDetectedPasswordFileOption(); + result = performInstanceConversionUsingVddk(sourceInstance, originalVMName, temporaryConvertPath, + vddkLibDir, serverResource.getLibguestfsBackend(), vddkTransports, configuredVddkThumbprint, + timeout, verboseModeEnabled, extraParams, temporaryConvertUuid, passwordOption); + } else { + logger.info("({}) Using OVF-based conversion (export + local convert)", originalVMName); + String sourceOVFDirPath; + if (cmd.getExportOvfToConversionLocation()) { + String exportInstanceOVAUrl = getExportInstanceOVAUrl(sourceInstance, originalVMName); + + if (StringUtils.isBlank(exportInstanceOVAUrl)) { + String err = String.format("Couldn't export OVA for the VM %s, due to empty url", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + + int noOfThreads = cmd.getThreadsCountToExportOvf(); + if (noOfThreads > 1 && !serverResource.ovfExportToolSupportsParallelThreads()) { + noOfThreads = 0; + } + ovfTemplateDirOnConversionLocation = UUID.randomUUID().toString(); + temporaryStoragePool.createFolder(ovfTemplateDirOnConversionLocation); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + ovfExported = exportOVAFromVMOnVcenter(exportInstanceOVAUrl, sourceOVFDirPath, noOfThreads, originalVMName, timeout); + + if (!ovfExported) { + String err = String.format("Export OVA for the VM %s failed", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); + return new Answer(cmd, false, err); + } + sourceOVFDirPath = String.format("%s%s/", sourceOVFDirPath, sourceInstanceName); + } else { + ovfTemplateDirOnConversionLocation = cmd.getTemplateDirOnConversionLocation(); + sourceOVFDirPath = String.format("%s/%s/", temporaryConvertPath, ovfTemplateDirOnConversionLocation); + } + + result = performInstanceConversion(originalVMName, sourceOVFDirPath, temporaryConvertPath, temporaryConvertUuid, + timeout, verboseModeEnabled, extraParams, serverResource); + } + if (!result) { - String err = String.format( - "The virt-v2v conversion for the OVF %s failed. Please check the agent logs " + - "for the virt-v2v output. Please try on a different kvm host which " + - "has a different virt-v2v version.", - ovfTemplateDirOnConversionLocation); - logger.error(String.format("(%s) %s", originalVMName, err)); + String err = String.format("Instance conversion failed for VM %s. Please check virt-v2v logs.", sourceInstanceName); + logger.error("({}) {}", originalVMName, err); return new Answer(cmd, false, err); } return new ConvertInstanceAnswer(cmd, temporaryConvertUuid); } catch (Exception e) { - String error = String.format("Error converting instance %s from %s, due to: %s", - sourceInstanceName, sourceHypervisorType, e.getMessage()); - logger.error(String.format("(%s) %s", originalVMName, error), e); + String error = String.format("Error converting instance %s from %s, due to: %s", sourceInstanceName, sourceHypervisorType, e.getMessage()); + logger.error("({}) {}", originalVMName, error, e); cleanupSecondaryStorage = true; return new Answer(cmd, false, error); } finally { @@ -275,4 +298,198 @@ public class LibvirtConvertInstanceCommandWrapper extends CommandWrapper/dev/null | " + + "openssl x509 -fingerprint -sha1 -noout", endpoint); + + Script script = new Script("/bin/bash", timeout, logger); + script.add("-c"); + script.add(command); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + script.execute(parser); + + String output = parser.getLines(); + if (script.getExitValue() != 0) { + logger.error("({}) Failed to fetch vCenter thumbprint for {}", originalVMName, vcenterHost); + return null; + } + + String thumbprint = extractSha1Fingerprint(output); + if (StringUtils.isBlank(thumbprint)) { + logger.error("({}) Failed to parse vCenter thumbprint from output for {}", originalVMName, vcenterHost); + return null; + } + return thumbprint; + } + + private String extractSha1Fingerprint(String output) { + String parsedOutput = StringUtils.trimToEmpty(output); + if (StringUtils.isBlank(parsedOutput)) { + return null; + } + + for (String line : parsedOutput.split("\\R")) { + String trimmedLine = StringUtils.trimToEmpty(line); + if (StringUtils.isBlank(trimmedLine)) { + continue; + } + + Matcher matcher = SHA1_FINGERPRINT_PATTERN.matcher(trimmedLine); + if (matcher.find()) { + return matcher.group(1).toUpperCase(Locale.ROOT); + } + + // Fallback for raw fingerprint-only output. + if (trimmedLine.matches("(?i)[0-9a-f]{2}(:[0-9a-f]{2})+")) { + return trimmedLine.toUpperCase(Locale.ROOT); + } + } + return null; + } + + /** + * Build vpx:// URL for virt-v2v + * + * Format: + * vpx://user@vcenter/DC/cluster/host?no_verify=1 + */ + private String buildVpxUrl(RemoteInstanceTO vmwareInstance) { + + String vmName = vmwareInstance.getInstanceName(); + String vcenter = vmwareInstance.getVcenterHost(); + String username = vmwareInstance.getVcenterUsername(); + String datacenter = vmwareInstance.getDatacenterName(); + String cluster = vmwareInstance.getClusterName(); + String host = vmwareInstance.getHostName(); + + String encodedUsername = encodeUsername(username); + + StringBuilder url = new StringBuilder(); + url.append("vpx://") + .append(encodedUsername) + .append("@") + .append(vcenter) + .append("/") + .append(datacenter); + + if (StringUtils.isNotBlank(cluster)) { + url.append("/").append(cluster); + } + + if (StringUtils.isNotBlank(host)) { + url.append("/").append(host); + } + + url.append("?no_verify=1"); + + logger.info("({}) Using VPX URL: {}", vmName, url); + return url.toString(); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java index 6d918435277..ab6bdd6135d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVolumesOnStorageCommandWrapper.java @@ -52,7 +52,7 @@ import java.util.stream.Collectors; public final class LibvirtGetVolumesOnStorageCommandWrapper extends CommandWrapper { static final List STORAGE_POOL_TYPES_SUPPORTED_BY_QEMU_IMG = Arrays.asList(StoragePoolType.NetworkFilesystem, - StoragePoolType.Filesystem, StoragePoolType.RBD); + StoragePoolType.Filesystem, StoragePoolType.RBD, StoragePoolType.SharedMountPoint); @Override public Answer execute(final GetVolumesOnStorageCommand command, final LibvirtComputingResource libvirtComputingResource) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java index 43607edc53a..f54918bbc22 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java @@ -259,6 +259,12 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper 0 && sleeptime > migrateWait * 1000) { DomainState state = null; try { @@ -306,8 +310,6 @@ public final class LibvirtMigrateCommandWrapper extends CommandWrapper 0 && sleeptime > migratePauseAfter) { DomainState state = null; try { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index e74923b281f..5a7d6d2c203 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -34,6 +34,7 @@ import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.utils.script.Script; +import org.apache.commons.lang3.StringUtils; @ResourceWrapper(handles = ReadyCommand.class) public final class LibvirtReadyCommandWrapper extends CommandWrapper { @@ -50,6 +51,9 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper restoreVolumePools = command.getRestoreVolumePools(); List restoreVolumePaths = command.getRestoreVolumePaths(); Integer mountTimeout = command.getMountTimeout() * 1000; - int timeout = command.getWait(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : serverResource.getCmdsTimeout(); KVMStoragePoolManager storagePoolMgr = serverResource.getStoragePoolMgr(); List backupFiles = command.getBackupFiles(); @@ -84,9 +99,9 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper(vmName, command.getVmState()), mountDirectory, timeout); } else if (Boolean.TRUE.equals(vmExists)) { restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); @@ -143,7 +158,7 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout) { String bkpPath; String volumeUuid; try { bkpPath = getBackupPath(mountDirectory, backupPath, backupFile, diskType); - volumeUuid = volumePath.substring(volumePath.lastIndexOf(File.separator) + 1); + volumeUuid = getVolumeUuidFromPath(volumePath, volumePool); verifyBackupFile(bkpPath, volumeUuid); - if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true)) { + if (!replaceVolumeWithBackup(storagePoolMgr, volumePool, volumePath, bkpPath, timeout, true, size)) { throw new CloudRuntimeException(String.format("Unable to restore contents from the backup volume [%s].", volumeUuid)); } @@ -247,42 +262,66 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper volumePools = command.getVolumePools(); final List volumePaths = command.getVolumePaths(); KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); + int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout(); List diskPaths = new ArrayList<>(); if (Objects.nonNull(volumePaths)) { @@ -81,7 +82,7 @@ public class LibvirtTakeBackupCommandWrapper extends CommandWrapper result = Script.executePipedCommands(commands, libvirtComputingResource.getCmdsTimeout()); + Pair result = Script.executePipedCommands(commands, timeout); if (result.first() != 0) { logger.debug("Failed to take VM backup: " + result.second()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java index f5bfd898a4f..8cf6de68f95 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/IscsiAdmStoragePool.java @@ -208,12 +208,12 @@ public class IscsiAdmStoragePool implements KVMStoragePool { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java index 8dd2116e123..3e35ed9476b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePool.java @@ -33,35 +33,33 @@ import com.cloud.storage.Storage.StoragePoolType; public interface KVMStoragePool { - public static final long HeartBeatUpdateTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); - public static final long HeartBeatUpdateFreq = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); - public static final long HeartBeatUpdateMaxTries = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_MAX_TRIES); - public static final long HeartBeatUpdateRetrySleep = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_RETRY_SLEEP); - public static final long HeartBeatCheckerTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); + long HeartBeatUpdateTimeoutInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HEARTBEAT_UPDATE_TIMEOUT); + long HeartBeatUpdateFreqInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_UPDATE_FREQUENCY); + long HeartBeatCheckerTimeoutInMs = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_HEARTBEAT_CHECKER_TIMEOUT); - public default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { + default KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, Long usableSize, byte[] passphrase) { return createPhysicalDisk(volumeUuid, format, provisioningType, size, passphrase); } - public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); + KVMPhysicalDisk createPhysicalDisk(String volumeUuid, PhysicalDiskFormat format, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); - public KVMPhysicalDisk createPhysicalDisk(String volumeUuid, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); + KVMPhysicalDisk createPhysicalDisk(String volumeUuid, Storage.ProvisioningType provisioningType, long size, byte[] passphrase); - public boolean connectPhysicalDisk(String volumeUuid, Map details); + boolean connectPhysicalDisk(String volumeUuid, Map details); - public KVMPhysicalDisk getPhysicalDisk(String volumeUuid); + KVMPhysicalDisk getPhysicalDisk(String volumeUuid); - public boolean disconnectPhysicalDisk(String volumeUuid); + boolean disconnectPhysicalDisk(String volumeUuid); - public boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format); + boolean deletePhysicalDisk(String volumeUuid, Storage.ImageFormat format); - public List listPhysicalDisks(); + List listPhysicalDisks(); - public String getUuid(); + String getUuid(); - public long getCapacity(); + long getCapacity(); - public long getUsed(); + long getUsed(); default Long getCapacityIops() { return null; @@ -71,51 +69,51 @@ public interface KVMStoragePool { return null; } - public long getAvailable(); + long getAvailable(); - public boolean refresh(); + boolean refresh(); - public boolean isExternalSnapshot(); + boolean isExternalSnapshot(); - public String getLocalPath(); + String getLocalPath(); - public String getSourceHost(); + String getSourceHost(); - public String getSourceDir(); + String getSourceDir(); - public int getSourcePort(); + int getSourcePort(); - public String getAuthUserName(); + String getAuthUserName(); - public String getAuthSecret(); + String getAuthSecret(); - public StoragePoolType getType(); + StoragePoolType getType(); - public boolean delete(); + boolean delete(); PhysicalDiskFormat getDefaultFormat(); - public boolean createFolder(String path); + boolean createFolder(String path); - public boolean supportsConfigDriveIso(); + boolean supportsConfigDriveIso(); - public Map getDetails(); + Map getDetails(); default String getLocalPathFor(String relativePath) { return String.format("%s%s%s", getLocalPath(), File.separator, relativePath); } - public boolean isPoolSupportHA(); + boolean isPoolSupportHA(); - public String getHearthBeatPath(); + String getHearthBeatPath(); - public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); + String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation); - public String getStorageNodeId(); + String getStorageNodeId(); - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host); + Boolean hasHeartBeat(HAStoragePool pool, HostTO host); - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration); + Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration); default LibvirtVMDef.DiskDef.BlockIOSize getSupportedLogicalBlockSize() { return null; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java index 6665cf625e2..35cc864268c 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStoragePoolManager.java @@ -289,6 +289,7 @@ public class KVMStoragePoolManager { if (pool instanceof LibvirtStoragePool) { addPoolDetails(uuid, (LibvirtStoragePool) pool); + ((LibvirtStoragePool) pool).setType(type); } return pool; @@ -390,6 +391,9 @@ public class KVMStoragePoolManager { private synchronized KVMStoragePool createStoragePool(String name, String host, int port, String path, String userInfo, StoragePoolType type, Map details, boolean primaryStorage) { StorageAdaptor adaptor = getStorageAdaptor(type); KVMStoragePool pool = adaptor.createStoragePool(name, host, port, path, userInfo, type, details, primaryStorage); + if (pool instanceof LibvirtStoragePool) { + ((LibvirtStoragePool) pool).setType(type); + } // LibvirtStorageAdaptor-specific statement if (pool.isPoolSupportHA() && primaryStorage) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 030d9747d6c..f95ebff5326 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -185,6 +185,8 @@ public class KVMStorageProcessor implements StorageProcessor { private int incrementalSnapshotTimeout; + private int incrementalSnapshotRetryRebaseWait; + private static final String CHECKPOINT_XML_TEMP_DIR = "/tmp/cloudstack/checkpointXMLs"; private static final String BACKUP_XML_TEMP_DIR = "/tmp/cloudstack/backupXMLs"; @@ -252,6 +254,7 @@ public class KVMStorageProcessor implements StorageProcessor { _cmdsTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.CMDS_TIMEOUT) * 1000; incrementalSnapshotTimeout = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_TIMEOUT) * 1000; + incrementalSnapshotRetryRebaseWait = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.INCREMENTAL_SNAPSHOT_RETRY_REBASE_WAIT) * 1000; return true; } @@ -2044,7 +2047,7 @@ public class KVMStorageProcessor implements StorageProcessor { try { Thread.sleep(10000); } catch (InterruptedException e) { - throw new CloudRuntimeException(e); + logger.trace("Thread that was tracking the progress for backup of VM [{}] was interrupted. Ignoring.", vmName); } } @@ -2093,8 +2096,25 @@ public class KVMStorageProcessor implements StorageProcessor { QemuImg qemuImg = new QemuImg(wait); qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); } catch (LibvirtException | QemuImgException e) { - logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); - throw new CloudRuntimeException(e); + if (!StringUtils.contains(e.getMessage(), "Is another process using the image")) { + logger.error("Exception while rebasing incremental snapshot [{}] due to: [{}].", snapshotName, e.getMessage(), e); + throw new CloudRuntimeException(e); + } + retryRebase(snapshotName, wait, e, snapshotFile, parentSnapshotFile); + } + } + + private void retryRebase(String snapshotName, int wait, Exception e, QemuImgFile snapshotFile, QemuImgFile parentSnapshotFile) { + logger.warn("Libvirt still has not released the lock, will wait [{}] milliseconds and try again later.", incrementalSnapshotRetryRebaseWait); + try { + Thread.sleep(incrementalSnapshotRetryRebaseWait); + QemuImg qemuImg = new QemuImg(wait); + qemuImg.rebase(snapshotFile, parentSnapshotFile, PhysicalDiskFormat.QCOW2.toString(), false); + } catch (LibvirtException | QemuImgException | InterruptedException ex) { + logger.error("Unable to rebase snapshot [{}].", snapshotName, ex); + CloudRuntimeException cre = new CloudRuntimeException(ex); + cre.addSuppressed(e); + throw cre; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java index ab39f7bc6ff..910f0eb15e0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java @@ -31,6 +31,7 @@ import org.apache.cloudstack.utils.qemu.QemuImg.PhysicalDiskFormat; import com.cloud.agent.api.to.HostTO; import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.ha.HighAvailabilityManager; import com.cloud.hypervisor.kvm.resource.KVMHABase.HAStoragePool; import com.cloud.storage.Storage; import com.cloud.storage.Storage.StoragePoolType; @@ -320,29 +321,38 @@ public class LibvirtStoragePool implements KVMStoragePool { @Override public boolean isPoolSupportHA() { - return type == StoragePoolType.NetworkFilesystem; + return HighAvailabilityManager.LIBVIRT_STORAGE_POOL_TYPES_WITH_HA_SUPPORT.contains(type); } public String getHearthBeatPath() { - if (type == StoragePoolType.NetworkFilesystem) { + if (StoragePoolType.NetworkFilesystem.equals(type)) { String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - return Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); + String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); + if (scriptPath == null) { + throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); + } + return scriptPath; + } else if (StoragePoolType.SharedMountPoint.equals(type)) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); + if (scriptPath == null) { + throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); + } + return scriptPath; } return null; } public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String hostPrivateIp, boolean hostValidation) { - Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeout, logger); + Script cmd = new Script(primaryStoragePool.getPool().getHearthBeatPath(), HeartBeatUpdateTimeoutInMs, logger); cmd.add("-i", primaryStoragePool.getPoolIp()); cmd.add("-p", primaryStoragePool.getPoolMountSourcePath()); cmd.add("-m", primaryStoragePool.getMountDestPath()); if (hostValidation) { cmd.add("-h", hostPrivateIp); - } - - if (!hostValidation) { + } else { cmd.add("-c"); } @@ -360,54 +370,58 @@ public class LibvirtStoragePool implements KVMStoragePool { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { - boolean validResult = false; + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { String hostIp = host.getPrivateNetwork().getIp(); - Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeout, logger); + Script cmd = new Script(getHearthBeatPath(), HeartBeatCheckerTimeoutInMs, logger); cmd.add("-i", pool.getPoolIp()); cmd.add("-p", pool.getPoolMountSourcePath()); cmd.add("-m", pool.getMountDestPath()); cmd.add("-h", hostIp); cmd.add("-r"); - cmd.add("-t", String.valueOf(HeartBeatUpdateFreq / 1000)); + cmd.add("-t", String.valueOf(HeartBeatUpdateFreqInMs / 1000)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); - logger.debug(String.format("Checking heart beat with KVMHAChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, - pool.getPoolIp())); + logger.debug("Checking heart beat for host IP {} with KVMHAChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn(String.format("Checking heart beat with KVMHAChecker command [%s] returned [%s]. [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), - result, parsedLine, hostIp)); + logger.warn("Checking heart beat for host IP {} with KVMHAChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); + return false; } else { - validResult = true; + logger.debug("Checking heart beat for host IP {} with KVMHAChecker command [{}] succeeded.", hostIp, cmd.toString()); + return true; } - return validResult; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + String hostIp = host.getPrivateNetwork().getIp(); Script cmd = new Script(vmActivityCheckPath, activityScriptTimeout.getStandardSeconds(), logger); cmd.add("-i", pool.getPoolIp()); cmd.add("-p", pool.getPoolMountSourcePath()); cmd.add("-m", pool.getMountDestPath()); - cmd.add("-h", host.getPrivateNetwork().getIp()); + cmd.add("-h", hostIp); cmd.add("-u", volumeUUIDListString); - cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000))); + cmd.add("-t", String.valueOf(System.currentTimeMillis() / 1000)); cmd.add("-d", String.valueOf(duration)); OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser(); String result = cmd.execute(parser); String parsedLine = parser.getLine(); - logger.debug(String.format("Checking heart beat with KVMHAVMActivityChecker [{command=\"%s\", result: \"%s\", log: \"%s\", pool: \"%s\"}].", cmd.toString(), result, parsedLine, pool.getPoolIp())); + logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker [{command=\"{}\", result: \"{}\", log: \"{}\", pool: \"{}\"}].", hostIp, cmd.toString(), result, parsedLine, pool.getPoolIp()); if (result == null && parsedLine.contains("DEAD")) { - logger.warn(String.format("Checking heart beat with KVMHAVMActivityChecker command [%s] returned [%s]. It is [%s]. It may cause a shutdown of host IP [%s].", cmd.toString(), result, parsedLine, host.getPrivateNetwork().getIp())); + logger.warn("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] returned [{}]. It may cause a shutdown of the host.", hostIp, cmd.toString(), parsedLine); return false; } else { + logger.debug("Checking VM activity for host IP {} with KVMHAVMActivityChecker command [{}] succeeded.", hostIp, cmd.toString()); return true; } } + + public void setType(StoragePoolType type) { + this.type = type; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java index 229481b1f79..df9986c9919 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIPool.java @@ -225,13 +225,13 @@ public class MultipathSCSIPool implements KVMStoragePool { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, - String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, + String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java index e336e0e5a54..82bc35f009e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java @@ -27,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Predicate; import com.cloud.agent.api.PrepareStorageClientCommand; import org.apache.cloudstack.storage.datastore.client.ScaleIOGatewayClient; @@ -644,18 +645,30 @@ public class ScaleIOStorageAdaptor implements StorageAdaptor { // Assuming SDC service is started, add mdms String mdms = details.get(ScaleIOGatewayClient.STORAGE_POOL_MDMS); String[] mdmAddresses = mdms.split(","); - if (mdmAddresses.length > 0) { - if (ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(true, getSDCDetails(details), "MDM added, no need to prepare the SDC client"); - } - - ScaleIOUtil.addMdms(mdmAddresses); - if (!ScaleIOUtil.isMdmPresent(mdmAddresses[0])) { - return new Ternary<>(false, null, "Failed to add MDMs"); + // remove MDMs already present in the config and added to the SDC + String[] mdmAddressesToAdd = Arrays.stream(mdmAddresses) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + // if all MDMs are already in the config and added to the SDC + if (mdmAddressesToAdd.length < 1 && mdmAddresses.length > 0) { + String msg = String.format("MDMs %s of the storage pool %s are already added", mdms, uuid); + logger.debug(msg); + return new Ternary<>(true, getSDCDetails(details), msg); + } else if (mdmAddressesToAdd.length > 0) { + ScaleIOUtil.addMdms(mdmAddressesToAdd); + String[] missingMdmAddresses = Arrays.stream(mdmAddressesToAdd) + .filter(Predicate.not(ScaleIOUtil::isMdmPresent)) + .toArray(String[]::new); + if (missingMdmAddresses.length > 0) { + String msg = String.format("Failed to add MDMs %s of the storage pool %s", String.join(", ", missingMdmAddresses), uuid); + logger.debug(msg); + return new Ternary<>(false, null, msg); } else { - logger.debug(String.format("MDMs %s added to storage pool %s", mdms, uuid)); + logger.debug("MDMs {} of the storage pool {} are added", mdmAddressesToAdd, uuid); applyMdmsChangeWaitTime(details); } + } else { + return new Ternary<>(false, getSDCDetails(details), "No MDM addresses provided"); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java index e8243c3f7cf..fc512ff94a9 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStoragePool.java @@ -236,12 +236,12 @@ public class ScaleIOStoragePool implements KVMStoragePool { } @Override - public Boolean checkingHeartBeat(HAStoragePool pool, HostTO host) { + public Boolean hasHeartBeat(HAStoragePool pool, HostTO host) { return null; } @Override - public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { + public Boolean hasVmActivity(HAStoragePool pool, HostTO host, Duration activityScriptTimeout, String volumeUUIDListString, String vmActivityCheckPath, long duration) { return null; } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java index 59ea720328f..3fbb5340fcc 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAConfig.java @@ -19,38 +19,37 @@ package org.apache.cloudstack.kvm.ha; import org.apache.cloudstack.framework.config.ConfigKey; -public class KVMHAConfig { +public interface KVMHAConfig { - public static final ConfigKey KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10", + ConfigKey KvmHAHealthCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.health.check.timeout", "10", "The maximum length of time, in seconds, expected for an health check to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60", + ConfigKey KvmHAActivityCheckTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.timeout", "60", "The maximum length of time, in seconds, expected for an activity check to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60", + ConfigKey KvmHAActivityCheckInterval = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.interval", "60", "The interval, in seconds, between activity checks.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10", + ConfigKey KvmHAActivityCheckMaxAttempts = new ConfigKey<>("Advanced", Long.class, "kvm.ha.activity.check.max.attempts", "10", "The maximum number of activity check attempts to perform before deciding to recover or degrade a resource.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7", + ConfigKey KvmHAActivityCheckFailureThreshold = new ConfigKey<>("Advanced", Double.class, "kvm.ha.activity.check.failure.ratio", "0.7", "The activity check failure threshold ratio. This is used with the activity check maximum attempts for deciding to recover or degrade a resource. For most environments, please keep this value above 0.5.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300", + ConfigKey KvmHADegradedMaxPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.degraded.max.period", "300", "The maximum length of time, in seconds, a resource can be in degraded state where only health checks are performed.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60", + ConfigKey KvmHARecoverTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.timeout", "60", "The maximum length of time, in seconds, expected for a recovery operation to complete.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600", + ConfigKey KvmHARecoverWaitPeriod = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.wait.period", "600", "The maximum length of time, in seconds, to wait for a resource to recover.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1", + ConfigKey KvmHARecoverAttemptThreshold = new ConfigKey<>("Advanced", Long.class, "kvm.ha.recover.failure.threshold", "1", "The maximum recovery attempts to be made for a resource, after which the resource is fenced. The recovery counter resets when a health check passes for a resource.", true, ConfigKey.Scope.Cluster); - public static final ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", + ConfigKey KvmHAFenceTimeout = new ConfigKey<>("Advanced", Long.class, "kvm.ha.fence.timeout", "60", "The maximum length of time, in seconds, expected for a fence operation to complete.", true, ConfigKey.Scope.Cluster); - } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java index b937be5265b..f0b5cfc337d 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHAProvider.java @@ -68,17 +68,18 @@ public final class KVMHAProvider extends HAAbstractHostProvider implements HAPro @Override public boolean recover(Host r) throws HARecoveryException { + logger.debug("Recover the host {}", r); try { - if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ + if (outOfBandManagementService.isOutOfBandManagementEnabled(r)) { final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); return resp.getSuccess(); } else { logger.warn("OOBM recover operation failed for the host {}", r); return false; } - } catch (Exception e){ + } catch (Exception e) { logger.warn("OOBM service is not configured or enabled for this host {} error is {}", r, e.getMessage()); - throw new HARecoveryException(String.format(" OOBM service is not configured or enabled for this host %s", r), e); + throw new HARecoveryException(String.format("OOBM service is not configured or enabled for this host %s", r), e); } } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java index 31f87d7e044..af7441c4fd2 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/kvm/ha/KVMHostActivityChecker.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.kvm.ha; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.CheckOnHostAnswer; import com.cloud.agent.api.CheckOnHostCommand; import com.cloud.agent.api.CheckVMActivityOnStoragePoolCommand; import com.cloud.dc.dao.ClusterDao; @@ -61,7 +62,7 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Inject private AgentManager agentMgr; @Inject - private PrimaryDataStoreDao storagePool; + private PrimaryDataStoreDao storagePoolDao; @Inject private StorageManager storageManager; @Inject @@ -70,11 +71,11 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Override public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException { try { - return isVMActivityOnHost(r, suspectTime); + return hasVMActivityOnHost(r, suspectTime); } catch (HACheckerException e) { - //Re-throwing the exception to avoid poluting the 'HACheckerException' already thrown + //Re-throwing the exception to avoid polluting the 'HACheckerException' already thrown throw e; - } catch (Exception e){ + } catch (Exception e) { String message = String.format("Operation timed out, probably the %s is not reachable.", r.toString()); logger.warn(message, e); throw new HACheckerException(message, e); @@ -83,82 +84,115 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck @Override public boolean isHealthy(Host r) { - return isAgentActive(r); + return isHostAgentUp(r); } - private boolean isAgentActive(Host agent) { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { - throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType())); + private boolean isHostAgentUp(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", host.getHypervisorType())); } - Status hostStatus = Status.Unknown; - Status neighbourStatus = Status.Unknown; - final CheckOnHostCommand cmd = new CheckOnHostCommand(agent, HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value()); - try { - logger.debug(String.format("Checking %s status...", agent.toString())); - Answer answer = agentMgr.easySend(agent.getId(), cmd); - if (answer != null) { - hostStatus = answer.getResult() ? Status.Down : Status.Up; - logger.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus)); - if ( hostStatus == Status.Up ){ - return true; + Status hostStatus = getHostAgentStatus(host); + + logger.debug("{} has the status [{}].", host.toString(), hostStatus); + return hostStatus == Status.Up; + } + + public Status getHostAgentStatus(Host host) { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + return null; + } + + Status hostStatusFromItself = checkHostStatusWithSameHost(host); + if (hostStatusFromItself == Status.Up) { + return Status.Up; + } + + Status hostStatusFromNeighbour = checkHostStatusWithNeighbourHosts(host); + Status hostStatus = hostStatusFromItself; + if (hostStatusFromNeighbour == Status.Up && (hostStatusFromItself == Status.Disconnected || hostStatusFromItself == Status.Down)) { + hostStatus = Status.Disconnected; + } + if (hostStatusFromNeighbour == Status.Down && (hostStatusFromItself == Status.Disconnected || hostStatusFromItself == Status.Down)) { + hostStatus = Status.Down; + } + + logger.debug("HA: HOST is ineligible legacy state {} for host {}", hostStatus, host); + return hostStatus; + } + + private Status checkHostStatusWithSameHost(Host host) { + Status hostStatus; + boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); + final CheckOnHostCommand cmd = new CheckOnHostCommand(host, reportFailureIfOneStorageIsDown); + try { + logger.debug("Checking {} status...", host.toString()); + Answer answer = agentMgr.easySend(host.getId(), cmd); + if (answer != null) { + if (answer.getResult()) { + hostStatus = ((CheckOnHostAnswer)answer).isAlive() ? Status.Up : Status.Down; + } else { + logger.debug("{} is not active according to itself, details: {}.", host.toString(), answer.getDetails()); + hostStatus = Status.Down; } - } - else { - logger.debug(String.format("Setting %s to \"Disconnected\" status.", agent.toString())); + logger.debug("{} has the status [{}].", host.toString(), hostStatus); + } else { + logger.debug("Setting {} to \"Disconnected\" status.", host.toString()); hostStatus = Status.Disconnected; } } catch (Exception e) { - logger.warn(String.format("Failed to send command CheckOnHostCommand to %s.", agent.toString()), e); + logger.warn("Failed to send command CheckOnHostCommand to {}.", host.toString(), e); + hostStatus = Status.Disconnected; } - List neighbors = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); + return hostStatus; + } + + private Status checkHostStatusWithNeighbourHosts(Host host) { + Status hostStatusFromNeighbour = Status.Unknown; + boolean reportFailureIfOneStorageIsDown = HighAvailabilityManager.KvmHAFenceHostIfHeartbeatFailsOnStorage.value(); + final CheckOnHostCommand cmd = new CheckOnHostCommand(host, reportFailureIfOneStorageIsDown); + List neighbors = resourceManager.listHostsInClusterByStatus(host.getClusterId(), Status.Up); for (HostVO neighbor : neighbors) { - if (neighbor.getId() == agent.getId() || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { + if (neighbor.getId() == host.getId() + || (neighbor.getHypervisorType() != Hypervisor.HypervisorType.KVM && neighbor.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { continue; } try { - logger.debug(String.format("Investigating %s via neighbouring %s.", agent.toString(), neighbor.toString())); - + logger.debug("Investigating {} via neighboring {}.", host.toString(), neighbor.toString()); Answer answer = agentMgr.easySend(neighbor.getId(), cmd); if (answer != null) { - neighbourStatus = answer.getResult() ? Status.Down : Status.Up; - - logger.debug(String.format("Neighbouring %s returned status [%s] for the investigated %s.", neighbor.toString(), neighbourStatus, agent.toString())); - - if (neighbourStatus == Status.Up) { - break; + if (answer.getResult()) { + hostStatusFromNeighbour = ((CheckOnHostAnswer)answer).isAlive() ? Status.Up : Status.Down; + logger.debug("Neighboring {} returned status [{}] for the investigated {}.", neighbor.toString(), hostStatusFromNeighbour, host.toString()); + if (hostStatusFromNeighbour == Status.Up) { + return hostStatusFromNeighbour; + } + } else { + logger.debug("{} is not active according to neighbor {}, details: {}.", host.toString(), neighbor.toString(), answer.getDetails()); } } else { - logger.debug(String.format("Neighbouring %s is Disconnected.", neighbor.toString())); + logger.debug("Neighboring {} is Disconnected.", neighbor.toString()); } } catch (Exception e) { - logger.warn(String.format("Failed to send command CheckOnHostCommand to %s.", neighbor.toString()), e); + logger.warn("Failed to send command CheckOnHostCommand to neighbor {}.", neighbor.toString(), e); } } - if (neighbourStatus == Status.Up && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Disconnected; - } - if (neighbourStatus == Status.Down && (hostStatus == Status.Disconnected || hostStatus == Status.Down)) { - hostStatus = Status.Down; - } - logger.debug(String.format("%s has the status [%s].", agent.toString(), hostStatus)); - - return hostStatus == Status.Up; + return hostStatusFromNeighbour; } - private boolean isVMActivityOnHost(Host agent, DateTime suspectTime) throws HACheckerException { - if (agent.getHypervisorType() != Hypervisor.HypervisorType.KVM && agent.getHypervisorType() != Hypervisor.HypervisorType.LXC) { - throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", agent.getHypervisorType())); + private boolean hasVMActivityOnHost(Host host, DateTime suspectTime) throws HACheckerException { + if (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC) { + throw new IllegalStateException(String.format("Calling KVM investigator for non KVM Host of type [%s].", host.getHypervisorType())); } boolean activityStatus = true; - HashMap> poolVolMap = getVolumeUuidOnHost(agent); - for (StoragePool pool : poolVolMap.keySet()) { - activityStatus = verifyActivityOfStorageOnHost(poolVolMap, pool, agent, suspectTime, activityStatus); + HashMap> poolVolumeMap = getStoragePoolAndVolumeInfoOnHost(host); + for (StoragePool pool : poolVolumeMap.keySet()) { + activityStatus = verifyActivityOfStorageOnHost(poolVolumeMap, pool, host, suspectTime, activityStatus); if (!activityStatus) { - logger.warn("It seems that the storage pool [{}] does not have activity on {}.", pool, agent); + logger.warn("It seems that the storage pool [{}] does not have activity on {}.", pool, host); break; } } @@ -166,66 +200,64 @@ public class KVMHostActivityChecker extends AdapterBase implements ActivityCheck return activityStatus; } - protected boolean verifyActivityOfStorageOnHost(HashMap> poolVolMap, StoragePool pool, Host agent, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException { + protected boolean verifyActivityOfStorageOnHost(HashMap> poolVolMap, StoragePool pool, Host host, DateTime suspectTime, boolean activityStatus) throws HACheckerException, IllegalStateException { List volume_list = poolVolMap.get(pool); - final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(agent, pool, volume_list, suspectTime); + final CheckVMActivityOnStoragePoolCommand cmd = new CheckVMActivityOnStoragePoolCommand(host, pool, volume_list, suspectTime); - logger.debug("Checking VM activity for {} on storage pool [{}].", agent.toString(), pool); + logger.debug("Checking VM activity for {} on storage pool [{}].", host.toString(), pool); try { - Answer answer = storageManager.sendToPool(pool, getNeighbors(agent), cmd); - + Answer answer = storageManager.sendToPool(pool, getNeighbors(host), cmd); if (answer != null) { activityStatus = !answer.getResult(); - logger.debug("{} {} activity on storage pool [{}]", agent.toString(), activityStatus ? "has" : "does not have", pool); + logger.debug("{} {} activity on storage pool [{}]", host.toString(), activityStatus ? "has" : "does not have", pool); } else { - String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", agent.toString(), pool); + String message = String.format("Did not get a valid response for VM activity check for %s on storage pool [%s].", host.toString(), pool); logger.debug(message); throw new IllegalStateException(message); } - } catch (StorageUnavailableException e){ - String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool, agent); + } catch (StorageUnavailableException e) { + String message = String.format("Storage [%s] is unavailable to do the check, probably the %s is not reachable.", pool, host); logger.warn(message, e); throw new HACheckerException(message, e); } return activityStatus; } - private HashMap> getVolumeUuidOnHost(Host agent) { - List vm_list = vmInstanceDao.listByHostId(agent.getId()); - List volume_list = new ArrayList(); - for (VirtualMachine vm : vm_list) { + private HashMap> getStoragePoolAndVolumeInfoOnHost(Host host) { + List vmListOnHost = vmInstanceDao.listByHostId(host.getId()); + List volumeListOnHost = new ArrayList<>(); + for (VirtualMachine vm : vmListOnHost) { logger.debug("Retrieving volumes of VM [{}]...", vm); - List vm_volume_list = volumeDao.findByInstance(vm.getId()); - volume_list.addAll(vm_volume_list); + List volumeListOfVM = volumeDao.findByInstance(vm.getId()); + volumeListOnHost.addAll(volumeListOfVM); } - HashMap> poolVolMap = new HashMap>(); - for (Volume vol : volume_list) { - StoragePool sp = storagePool.findById(vol.getPoolId()); - logger.debug("Retrieving storage pool [{}] of volume [{}]...", sp, vol); - if (!poolVolMap.containsKey(sp)) { - List list = new ArrayList(); - list.add(vol); + HashMap> poolVolumeMap = new HashMap<>(); + for (Volume volume : volumeListOnHost) { + StoragePool pool = storagePoolDao.findById(volume.getPoolId()); + logger.debug("Retrieving storage pool [{}] of volume [{}]...", pool, volume); + if (!poolVolumeMap.containsKey(pool)) { + List volList = new ArrayList<>(); + volList.add(volume); - poolVolMap.put(sp, list); + poolVolumeMap.put(pool, volList); } else { - poolVolMap.get(sp).add(vol); + poolVolumeMap.get(pool).add(volume); } } - return poolVolMap; + return poolVolumeMap; } - public long[] getNeighbors(Host agent) { - List neighbors = new ArrayList(); - List cluster_hosts = resourceManager.listHostsInClusterByStatus(agent.getClusterId(), Status.Up); - logger.debug("Retrieving all \"Up\" hosts from cluster [{}]...", clusterDao.findById(agent.getClusterId())); - for (HostVO host : cluster_hosts) { - if (host.getId() == agent.getId() || (host.getHypervisorType() != Hypervisor.HypervisorType.KVM && host.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { + public long[] getNeighbors(Host host) { + List neighbors = new ArrayList<>(); + List clusterHosts = resourceManager.listHostsInClusterByStatus(host.getClusterId(), Status.Up); + logger.debug("Retrieving all \"Up\" hosts from cluster [{}]...", clusterDao.findById(host.getClusterId())); + for (HostVO clusterHost : clusterHosts) { + if (clusterHost.getId() == host.getId() || (clusterHost.getHypervisorType() != Hypervisor.HypervisorType.KVM && clusterHost.getHypervisorType() != Hypervisor.HypervisorType.LXC)) { continue; } - neighbors.add(host.getId()); + neighbors.add(clusterHost.getId()); } return ArrayUtils.toPrimitive(neighbors.toArray(new Long[neighbors.size()])); } - } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index b9629524007..b3bdafb7375 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -3133,7 +3133,7 @@ public class LibvirtComputingResourceTest { assertNotNull(wrapper); final Answer answer = wrapper.execute(command, libvirtComputingResourceMock); - assertTrue(answer.getResult()); + assertFalse(answer.getResult()); verify(libvirtComputingResourceMock, times(1)).getMonitor(); } @@ -5642,35 +5642,45 @@ public class LibvirtComputingResourceTest { Mockito.verify(vmDef, times(1)).addComp(any()); } - public void validateGetCurrentMemAccordingToMemBallooningWithoutMemBalooning(){ + @Test + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryWithoutMemBallooning(){ VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); - Mockito.when(vmTo.getType()).thenReturn(Type.User); LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); libvirtComputingResource.noMemBalloon = true; - long maxMemory = 2048; + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; - long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, maxMemory); - Assert.assertEquals(maxMemory, currentMemory); - Mockito.verify(vmTo, Mockito.times(0)).getMinRam(); + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(requestedMemory, currentMemory); } @Test - public void validateGetCurrentMemAccordingToMemBallooningWithtMemBalooning(){ + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryWithMemoryBallooning(){ LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); libvirtComputingResource.noMemBalloon = false; - long maxMemory = 2048; - long minMemory = ByteScaleUtils.mebibytesToBytes(64); - VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); Mockito.when(vmTo.getType()).thenReturn(Type.User); - Mockito.when(vmTo.getMinRam()).thenReturn(minMemory); + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; - long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, maxMemory); - Assert.assertEquals(ByteScaleUtils.bytesToKibibytes(minMemory), currentMemory); - Mockito.verify(vmTo).getMinRam(); + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(minMemory, currentMemory); } + @Test + public void getCurrentMemAccordingToMemBallooningTestValidateCurrentMemoryForSystemVms() { + LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); + libvirtComputingResource.noMemBalloon = false; + + VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); + Mockito.when(vmTo.getType()).thenReturn(Type.SecondaryStorageVm); + long requestedMemory = 1024 * 1024; + long minMemory = 512 * 1024; + + long currentMemory = libvirtComputingResource.getCurrentMemAccordingToMemBallooning(vmTo, requestedMemory, minMemory); + Assert.assertEquals(requestedMemory, currentMemory); + } @Test public void validateCreateGuestResourceDefWithVcpuMaxLimit(){ LibvirtComputingResource libvirtComputingResource = new LibvirtComputingResource(); @@ -7234,4 +7244,82 @@ public class LibvirtComputingResourceTest { libvirtComputingResourceSpy.getInterface(connMock, vmName, invalidMacAddress); } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreNotUpdatedWhenLibvirtVersionIsLessThanTheMinimum() throws LibvirtException { + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 8999; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, null, false); + Mockito.verify(domainMock, Mockito.never()).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreNotUpdatedWhenThereIsNoCapCapChangeAndNoCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, false); + Mockito.verify(domainMock, Mockito.never()).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertQuotaIsRemovedWhenThereIsCpuCapChangeAndNoCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(false); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, true); + Mockito.verify(domainMock, Mockito.times(1)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreUpdatedWhenThereIsNotCpuCapChangeAndCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); + double cpuQuotaPercentage = 0.03; + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(cpuQuotaPercentage); + Mockito.doReturn(new Pair<>(1000, 300L)).when(libvirtComputingResourceSpy).getPeriodAndQuota(cpuQuotaPercentage); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, false); + Mockito.verify(domainMock, Mockito.times(2)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void updateCpuQuotaAndPeriodTestAssertPeriodAndQuotaAreUpdatedWhenThereIsCpuCapChangeAndCpuLimitationIsApplied() throws LibvirtException { + Mockito.when(vmTO.isLimitCpuUse()).thenReturn(true); + double cpuQuotaPercentage = 0.03; + Mockito.when(vmTO.getCpuQuotaPercentage()).thenReturn(cpuQuotaPercentage); + Mockito.doReturn(new Pair<>(1000, 300L)).when(libvirtComputingResourceSpy).getPeriodAndQuota(cpuQuotaPercentage); + Mockito.when(domainMock.getName()).thenReturn("i-2-10-VM"); + libvirtComputingResourceSpy.hypervisorLibvirtVersion = 9000; + libvirtComputingResourceSpy.updateCpuQuotaAndPeriod(domainMock, vmTO, true); + Mockito.verify(domainMock, Mockito.times(2)).setSchedulerParameters(Mockito.any()); + } + + @Test + public void getPeriodAndQuotaTestAssertQuotaIsEqualToPeriodMultipliedByQuotaPercentage() { + double cpuQuotaPercentage = 0.3; + int expectedPeriod = CpuTuneDef.DEFAULT_PERIOD; + long expectedQuota = (long) (expectedPeriod * cpuQuotaPercentage); + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void getPeriodAndQuotaTestQuotaIsEqualToMinimumWhenRequired() { + double cpuQuotaPercentage = 0.03; + long expectedQuota = CpuTuneDef.MIN_QUOTA; + int expectedPeriod = (int) ((double) expectedQuota / cpuQuotaPercentage); + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } + + @Test + public void getPeriodAndQuotaTestPeriodIsEqualToMaximumWhenRequired() { + double cpuQuotaPercentage = 0.0003; + long expectedQuota = CpuTuneDef.MIN_QUOTA; + int expectedPeriod = CpuTuneDef.MAX_PERIOD; + Pair expectedResult = new Pair<>(expectedPeriod, expectedQuota); + Pair result = libvirtComputingResourceSpy.getPeriodAndQuota(cpuQuotaPercentage); + Assert.assertEquals(expectedResult, result); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java index 0060e1d7ed4..e6f63e852f8 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtGpuDefTest.java @@ -115,6 +115,145 @@ public class LibvirtGpuDefTest extends TestCase { assertTrue(gpuXml.contains("")); } + @Test + public void testGpuDef_withFullPciAddressDomainZero() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0000:00:02.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "0001:65:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withNvidiaStyleEightDigitDomain() { + // nvidia-smi reports PCI addresses with an 8-digit domain (e.g. "00000001:af:00.1"). + // generatePciXml must normalize it to the canonical 4-digit form "0x0001". + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "00000001:af:00.1", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withFullPciAddressVfNonZeroDomain() { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo vfGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "VF-Profile", + "VF-Profile", + "0002:81:00.3", + "10de", + "NVIDIA Corporation", + "1eb8", + "Tesla T4" + ); + gpuDef.defGpu(vfGpuInfo); + + String gpuXml = gpuDef.toString(); + + // Non-passthrough NVIDIA VFs should be unmanaged + assertTrue(gpuXml.contains("")); + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withLegacyShortBdfDefaultsDomainToZero() { + // Backward compatibility: short BDF with no domain segment must still + // produce a valid libvirt address with domain 0x0000. + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo pciGpuInfo = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + "af:00.0", + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(pciGpuInfo); + + String gpuXml = gpuDef.toString(); + + assertTrue(gpuXml.contains("
")); + } + + @Test + public void testGpuDef_withInvalidBusAddressThrows() { + String[] invalidAddresses = { + "notahex:00.0", // non-hex bus + "gg:00:02.0", // non-hex domain + "00:02:03:04", // too many colon-separated parts + "00", // missing slot/function + "00:02", // missing function (no dot) + "00:02.0.1", // extra dot in ss.f + }; + for (String addr : invalidAddresses) { + LibvirtGpuDef gpuDef = new LibvirtGpuDef(); + VgpuTypesInfo info = new VgpuTypesInfo( + GpuDevice.DeviceType.PCI, + "passthrough", + "passthrough", + addr, + "10de", + "NVIDIA Corporation", + "1b38", + "Tesla T4" + ); + gpuDef.defGpu(info); + try { + String ignored = gpuDef.toString(); + fail("Expected IllegalArgumentException for address: " + addr + " but got: " + ignored); + } catch (IllegalArgumentException e) { + assertTrue("Exception message should contain the bad address", + e.getMessage().contains(addr)); + } + } + } + @Test public void testGpuDef_withNullDeviceType() { LibvirtGpuDef gpuDef = new LibvirtGpuDef(); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java index 3cad9c27a68..74090723331 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckConvertInstanceCommandWrapperTest.java @@ -52,6 +52,7 @@ public class LibvirtCheckConvertInstanceCommandWrapperTest { @Test public void testCheckInstanceCommand_success() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(false); Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(true); Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); assertTrue(answer.getResult()); @@ -59,9 +60,33 @@ public class LibvirtCheckConvertInstanceCommandWrapperTest { @Test public void testCheckInstanceCommand_failure() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(false); Mockito.when(libvirtComputingResourceMock.hostSupportsInstanceConversion()).thenReturn(false); Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); assertFalse(answer.getResult()); assertTrue(StringUtils.isNotBlank(answer.getDetails())); } + + @Test + public void testCheckInstanceCommand_vddkSuccess() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(true); + Mockito.when(checkConvertInstanceCommandMock.getVddkLibDir()).thenReturn("/opt/vmware-vddk/vmware-vix-disklib-distrib"); + Mockito.when(libvirtComputingResourceMock.hostSupportsVddk("/opt/vmware-vddk/vmware-vix-disklib-distrib")).thenReturn(true); + + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + + assertTrue(answer.getResult()); + } + + @Test + public void testCheckInstanceCommand_vddkFailure() { + Mockito.when(checkConvertInstanceCommandMock.isUseVddk()).thenReturn(true); + Mockito.when(checkConvertInstanceCommandMock.getVddkLibDir()).thenReturn("/opt/vmware-vddk/vmware-vix-disklib-distrib"); + Mockito.when(libvirtComputingResourceMock.hostSupportsVddk("/opt/vmware-vddk/vmware-vix-disklib-distrib")).thenReturn(false); + + Answer answer = checkConvertInstanceCommandWrapper.execute(checkConvertInstanceCommandMock, libvirtComputingResourceMock); + + assertFalse(answer.getResult()); + assertTrue(StringUtils.isNotBlank(answer.getDetails())); + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java new file mode 100644 index 00000000000..a06a9c50bc1 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVirtualMachineCommandWrapperTest.java @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; +import org.libvirt.DomainInfo.DomainState; +import org.libvirt.LibvirtException; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.CheckVirtualMachineAnswer; +import com.cloud.agent.api.CheckVirtualMachineCommand; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.vm.VirtualMachine.PowerState; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtCheckVirtualMachineCommandWrapperTest { + + private static final String VM_NAME = "i-2-3-VM"; + + @Mock + private LibvirtComputingResource libvirtComputingResource; + @Mock + private LibvirtUtilitiesHelper libvirtUtilitiesHelper; + @Mock + private Connect conn; + @Mock + private Domain domain; + + private LibvirtCheckVirtualMachineCommandWrapper wrapper; + private CheckVirtualMachineCommand command; + + @Before + public void setUp() throws LibvirtException { + wrapper = new LibvirtCheckVirtualMachineCommandWrapper(); + command = new CheckVirtualMachineCommand(VM_NAME); + + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnectionByVmName(VM_NAME)).thenReturn(conn); + } + + @Test + public void testExecuteVmPoweredOnReturnsStateAndVncPort() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_RUNNING; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5900); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOn, answer.getState()); + assertEquals(Integer.valueOf(5900), answer.getVncPort()); + } + + @Test + public void testExecuteVmPausedReturnsPowerUnknown() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_PAUSED; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5901); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerUnknown, answer.getState()); + assertEquals(Integer.valueOf(5901), answer.getVncPort()); + } + + @Test + public void testExecuteVmPoweredOffReturnsStateWithNullVncPort() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOff); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOff, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteVmStateUnknownReturnsStateWithNullVncPort() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerUnknown); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerUnknown, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteVmPoweredOnWithNullVncPort() throws LibvirtException { + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainState.VIR_DOMAIN_RUNNING; + + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(null); + when(conn.domainLookupByName(VM_NAME)).thenReturn(domain); + when(domain.getInfo()).thenReturn(domainInfo); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertTrue(answer.getResult()); + assertEquals(PowerState.PowerOn, answer.getState()); + assertNull(answer.getVncPort()); + } + + @Test + public void testExecuteLibvirtExceptionOnGetConnectionReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("Connection refused"); + when(libvirtUtilitiesHelper.getConnectionByVmName(VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("Connection refused", answer.getDetails()); + } + + @Test + public void testExecuteLibvirtExceptionOnGetVncPortReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("VNC port error"); + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("VNC port error", answer.getDetails()); + } + + @Test + public void testExecuteLibvirtExceptionOnDomainLookupReturnsFailure() throws LibvirtException { + LibvirtException libvirtException = mock(LibvirtException.class); + when(libvirtException.getMessage()).thenReturn("Domain not found"); + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOn); + when(libvirtComputingResource.getVncPort(conn, VM_NAME)).thenReturn(5900); + when(conn.domainLookupByName(VM_NAME)).thenThrow(libvirtException); + + CheckVirtualMachineAnswer answer = (CheckVirtualMachineAnswer) wrapper.execute(command, libvirtComputingResource); + + assertFalse(answer.getResult()); + assertEquals("Domain not found", answer.getDetails()); + } + + @Test + public void testExecuteCallsGetLibvirtUtilitiesHelper() throws LibvirtException { + when(libvirtComputingResource.getVmState(conn, VM_NAME)).thenReturn(PowerState.PowerOff); + + wrapper.execute(command, libvirtComputingResource); + + verify(libvirtComputingResource).getLibvirtUtilitiesHelper(); + verify(libvirtUtilitiesHelper).getConnectionByVmName(VM_NAME); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java index 4d55ac2bc73..c30fa2f4948 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtConvertInstanceCommandWrapperTest.java @@ -18,6 +18,7 @@ // package com.cloud.hypervisor.kvm.resource.wrapper; +import java.nio.file.Files; import java.util.List; import java.util.UUID; @@ -189,4 +190,127 @@ public class LibvirtConvertInstanceCommandWrapperTest { Mockito.verify(script).add("-x"); Mockito.verify(script).add("-v"); } + + @Test + public void testPerformInstanceConversionUsingVddkUsesConfiguredLibguestfsBackend() { + RemoteInstanceTO remoteInstanceTO = Mockito.mock(RemoteInstanceTO.class); + Mockito.when(remoteInstanceTO.getVcenterHost()).thenReturn("vcenter.local"); + Mockito.when(remoteInstanceTO.getVcenterUsername()).thenReturn("administrator@vsphere.local"); + Mockito.when(remoteInstanceTO.getVcenterPassword()).thenReturn("secret"); + Mockito.when(remoteInstanceTO.getDatacenterName()).thenReturn("dc1"); + Mockito.when(remoteInstanceTO.getClusterName()).thenReturn("cluster1"); + Mockito.when(remoteInstanceTO.getHostName()).thenReturn("host1"); + Mockito.doReturn("28:19:A6:1C:90:ED:46:D7:1C:86:BC:F6:13:52:F0:B9:19:81:0D:81") + .when(convertInstanceCommandWrapper).getVcenterThumbprint(Mockito.anyString(), Mockito.anyLong(), Mockito.anyString()); + + try (MockedStatic filesMock = Mockito.mockStatic(Files.class); + MockedConstruction