Merge branch '4.22'

Bumps `ads.version` from 2.0.0.AM25 to 2.0.0.AM27.

Updates `org.apache.directory.server:apacheds-server-integ` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

Updates `org.apache.directory.server:apacheds-core-constants` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

Updates `org.apache.directory.server:apacheds-core-annotations` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

Updates `org.apache.directory.server:apacheds-core` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

Updates `org.apache.directory.server:apacheds-protocol-ldap` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

Updates `org.apache.directory.server:apacheds-jdbm-partition` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

Updates `org.apache.directory.server:apacheds-ldif-partition` from 2.0.0.AM25 to 2.0.0.AM27
- [Commits](https://github.com/apache/directory-server/compare/2.0.0.AM25...2.0.0.AM27)

---
updated-dependencies:
- dependency-name: org.apache.directory.server:apacheds-server-integ
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
- dependency-name: org.apache.directory.server:apacheds-core-constants
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
- dependency-name: org.apache.directory.server:apacheds-core-annotations
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
- dependency-name: org.apache.directory.server:apacheds-core
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
- dependency-name: org.apache.directory.server:apacheds-protocol-ldap
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
- dependency-name: org.apache.directory.server:apacheds-jdbm-partition
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
- dependency-name: org.apache.directory.server:apacheds-ldif-partition
  dependency-version: 2.0.0.AM27
  dependency-type: direct:development
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.asf.yaml

@@ -51,20 +51,20 @@ github:
 
   collaborators:
     - acs-robot
-    - rajujith
-    - GaOrtiga
-    - SadiJr
-    - winterhazel
     - gpordeus
     - hsato03
-    - bernardodemarco
-    - abh1sar
     - FelipeM525
+    - lucas-a-martins
+    - nicoschmdt
+    - abh1sar
+    - rosi-shapeblue
+    - sudo87
+    - erikbocks
 
   protected_branches: ~
 
 notifications:
-  commits:      commits@cloudstack.apache.org
+  commits: commits@cloudstack.apache.org
-  issues:       commits@cloudstack.apache.org
+  issues: commits@cloudstack.apache.org
   pullrequests: commits@cloudstack.apache.org
-  discussions:  users@cloudstack.apache.org
+  discussions: users@cloudstack.apache.org

.github/CODEOWNERS

@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class CloudStackPage(object):
+/plugins/storage/volume/linstor @rp-
-    def __init__():
+/plugins/storage/volume/storpool @slavkap
-        self.browser = None
+
+.pre-commit-config.yaml @jbampton
+/.github/linters/ @jbampton

.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+description: "Thank you for reporting a bug!"
+name: bug
+title: "[SHORT PROBLEM DESCRIPTION]"
+labels: bug, needs-triageing
+body:
+- type: markdown
+  attributes:
+    value: "## Welcome, please describe your problem below;"
+- type: textarea
+  attributes:
+    label: problem
+    value: The long description of your problem
+- type: markdown
+  attributes:
+    value: "## What versions of cloudstack and any infra components are you using"
+- type: textarea
+  attributes:
+    label: versions
+    value: The versions of ACS, hypervisors, storage, network etc..
+- type: textarea
+  attributes:
+    label: The steps to reproduce the bug
+    value: |
+      1.
+      2.
+      3.
+      ...
+- type: textarea
+  attributes:
+    label: "What to do about it?"

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+description: "Thank you for your new feature idea!"
+name: feature
+title: "[SHORT FUNCTIONAL DESCRIPTION]"
+labels:  new
+body:
+- type: textarea
+  attributes:
+    label: "The required feature described as a wish"
+    value: As a User/Admin/Operator I would like to , ... have the system make my morning coffee.

.github/dependabot.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "maven" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+    cooldown:
+      default-days: 7

.github/linters/.markdown-lint.yml

@@ -18,36 +18,21 @@
 # MD001/heading-increment Heading levels should only increment by one level at a time
 MD001: false
 
-# MD003/heading-style Heading style
-MD003: false
-
 # MD004/ul-style Unordered list style
 MD004: false
 
 # MD007/ul-indent Unordered list indentation
 MD007: false
 
-# MD009/no-trailing-spaces Trailing spaces
-MD009: false
-
 # MD010/no-hard-tabs Hard tabs
 MD010: false
 
-# MD012/no-multiple-blanks Multiple consecutive blank lines
-MD012: false
-
 # MD013/line-length Line length
 MD013: false
 
 # MD014/commands-show-output Dollar signs used before commands without showing output
 MD014: false
 
-# MD018/no-missing-space-atx No space after hash on atx style heading
-MD018: false
-
-# MD019/no-multiple-space-atx Multiple spaces after hash on atx style heading
-MD019: false
-
 # MD022/blanks-around-headings Headings should be surrounded by blank lines
 MD022: false
 
@@ -98,3 +83,6 @@ MD046: false
 
 # MD052/reference-links-images Reference links and images should use a label that is defined
 MD052: false
+
+# MD059/descriptive-link-text Link text should be descriptive
+MD059: false

.github/linters/.yamllint.yml

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+extends: default
+
+rules:
+  line-length:
+    max: 400  # Very forgiving for GitHub Actions and infrastructure files
+  indentation: disable  # Disable indentation checking for existing files
+  comments: disable  # Disable comment formatting checks
+  braces: disable
+  brackets: disable  # Disable bracket spacing checks
+  colons:
+    max-spaces-after: -1  # Allow any number of spaces after colon
+    max-spaces-before: 0
+  document-start: disable  # Many files don't have ---
+  truthy:
+    allowed-values: ['true', 'false', 'on', 'off', 'yes', 'no']

.github/linters/codespell.txt

@@ -4,6 +4,7 @@ acount
 actuall
 acuiring
 acumulate
+addin
 addreess
 addtion
 adminstrator
@@ -12,10 +13,8 @@ afrer
 afterall
 againt
 ags
-aktive
 algoritm
 allo
-alloacate
 allocted
 alocation
 alogrithm
@@ -65,6 +64,7 @@ bject
 boardcast
 bootstraper
 bu
+callin
 cant
 capabilites
 capablity
@@ -73,6 +73,7 @@ carrefully
 cavaet
 chaing
 checkd
+checkin
 childs
 choosen
 chould
@@ -93,7 +94,6 @@ confg
 configruation
 configuable
 conneciton
-connexion
 constrait
 constraits
 containg
@@ -101,9 +101,7 @@ contex
 continuesly
 contro
 controler
-controles
 controll
-convienient
 convinience
 coputer
 correcponding
@@ -158,13 +156,13 @@ differnet
 differnt
 direcotry
 directroy
-disale
 disbale
 discrepency
 disover
 dissapper
 dissassociated
 divice
+dockin
 doesn'
 doesnot
 doesnt
@@ -175,7 +173,6 @@ eanbled
 earch
 ect
 elemnt
-eles
 elments
 emmited
 enble
@@ -183,29 +180,23 @@ encryted
 enebled
 enmpty
 entires
-enviornment
 environmnet
 equivalant
 erro
 erronous
-everthing
 everytime
-excetion
-excption
 excute
 execept
 execption
+exects
 execut
 executeable
 exeeded
 exisitng
 exisits
-existin
 existsing
-exitting
 expcted
 expection
-explaination
 explicitely
 faield
 faild
@@ -218,7 +209,6 @@ fillled
 findout
 fisrt
 fo
-folowing
 fowarding
 frist
 fro
@@ -237,6 +227,7 @@ hanling
 happend
 hasing
 hasnt
+havin
 hda
 hostanme
 hould
@@ -256,20 +247,14 @@ implmeneted
 implmentation
 incase
 includeing
-incosistency
 indecates
-indien
 infor
 informations
 informaton
-infrastrcuture
 ingore
-inital
 initalize
 initator
-initilization
 inspite
-instace
 instal
 instnace
 intefaces
@@ -287,12 +272,8 @@ ist
 klunky
 lable
 leve
-lief
 limite
-linke
 listner
-lokal
-lokales
 maintainence
 maintenace
 maintenence
@@ -301,7 +282,6 @@ mambers
 manaully
 manuel
 maxium
-mehtod
 mergable
 mesage
 messge
@@ -311,7 +291,6 @@ minumum
 mis
 modifers
 mor
-mot
 mulitply
 multipl
 multple
@@ -325,7 +304,7 @@ nin
 nodel
 nome
 noone
-nowe
+notin
 numbe
 numer
 occured
@@ -378,6 +357,7 @@ propogate
 provison
 psudo
 pyhsical
+re-use
 readabilty
 readd
 reccuring
@@ -392,12 +372,9 @@ remaning
 remore
 remvoing
 renabling
-repeatly
 reponse
 reqest
 reqiured
-requieres
-requried
 reserv
 reserverd
 reseted
@@ -414,17 +391,15 @@ retriving
 retrun
 retuned
 returing
-re-use
 rever
 rocessor
+roperty
 runing
 runnign
 sate
 scalled
-scipt
 scirpt
 scrip
-seconadry
 seconday
 seesion
 sepcified
@@ -437,12 +412,10 @@ settig
 sevices
 shoul
 shoule
-sie
 signle
 simplier
 singature
 skiping
-snaphsot
 snpashot
 specied
 specifed
@@ -453,7 +426,6 @@ standy
 statics
 stickyness
 stil
-stip
 storeage
 strat
 streched
@@ -462,7 +434,6 @@ succesfull
 successfull
 suceessful
 suces
-sucessfully
 suiteable
 suppots
 suppport
@@ -495,7 +466,6 @@ uncompressible
 uneccessarily
 unexepected
 unexpect
-unknow
 unkonw
 unkown
 unneccessary
@@ -503,14 +473,12 @@ unparseable
 unrecoginized
 unsupport
 unxpected
-updat
 uptodate
 usera
 usign
 usin
 utlization
 vaidate
-valiate
 valule
 valus
 varibles
@@ -519,8 +487,6 @@ verfying
 verifing
 virutal
 visable
-wakup
 wil
 wit
-wll
 wth

.github/workflows/build.yml

@@ -30,18 +30,17 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
-      - name: Set up JDK 11
+      - name: Set up JDK 17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
-          java-version: '11'
+          distribution: 'temurin'
-          distribution: 'adopt'
+          java-version: '17'
-          architecture: x64
+          cache: 'maven'
-          cache: maven
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
           architecture: 'x64'

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ permissions:
 jobs:
   build:
     if: github.repository == 'apache/cloudstack'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       fail-fast: false
@@ -89,7 +89,10 @@ jobs:
                   smoke/test_nested_virtualization
                   smoke/test_set_sourcenat
                   smoke/test_webhook_lifecycle
-                  smoke/test_purge_expunged_vms",
+                  smoke/test_purge_expunged_vms
+                  smoke/test_extension_lifecycle
+                  smoke/test_extension_custom_action_lifecycle
+                  smoke/test_extension_custom",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
@@ -137,6 +140,7 @@ jobs:
                   smoke/test_vm_deployment_planner
                   smoke/test_vm_strict_host_tags
                   smoke/test_vm_schedule
+                  smoke/test_deploy_vgpu_enabled_vm
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm
@@ -164,7 +168,8 @@ jobs:
                   component/test_cpu_limits
                   component/test_cpu_max_limits
                   component/test_cpu_project_limits
-                  component/test_deploy_vm_userdata_multi_nic",
+                  component/test_deploy_vm_userdata_multi_nic
+                  component/test_deploy_vm_lease",
                 "component/test_egress_fw_rules
                   component/test_invalid_gw_nm
                   component/test_ip_reservation",
@@ -211,20 +216,19 @@ jobs:
                   smoke/test_list_volumes"]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
-      - name: Set up JDK
+      - name: Set up JDK 17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
-          java-version: '11'
+          distribution: 'temurin'
-          distribution: 'adopt'
+          java-version: '17'
-          architecture: x64
+          cache: 'maven'
-          cache: maven
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.10'
           architecture: 'x64'
@@ -232,7 +236,25 @@ jobs:
       - name: Install Build Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install -y git uuid-runtime genisoimage netcat ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools
+          sudo apt-get install -y git uuid-runtime genisoimage netcat-openbsd ipmitool build-essential libgcrypt20 libgpg-error-dev libgpg-error0 libopenipmi0 ipmitool libpython3-dev libssl-dev libffi-dev python3-openssl python3-dev python3-setuptools
+
+      - name: Setup IPMI Tool for CloudStack
+        run: |
+          # Create cloudstack-common directory if it doesn't exist
+          sudo mkdir -p /usr/share/cloudstack-common
+
+          # Copy ipmitool to cloudstack-common directory if it doesn't exist
+          if [ ! -f /usr/share/cloudstack-common/ipmitool ]; then
+            sudo cp /usr/bin/ipmitool /usr/share/cloudstack-common/ipmitool
+            sudo chmod 755 /usr/share/cloudstack-common/ipmitool
+          fi
+
+          # Create ipmitool-C3 wrapper script
+          sudo tee /usr/bin/ipmitool > /dev/null << 'EOF'
+          #!/bin/bash
+          /usr/share/cloudstack-common/ipmitool -C3 $@
+          EOF
+          sudo chmod 755 /usr/bin/ipmitool
 
       - name: Install Python dependencies
         run: |
@@ -271,7 +293,7 @@ jobs:
       - name: Setup Simulator Prerequisites
         run: |
           sudo python3 -m pip install --upgrade netaddr mysql-connector-python
-          python3 -m pip install --user --upgrade tools/marvin/dist/Marvin-*.tar.gz
+          python3 -m pip install --user --upgrade tools/marvin/dist/[mM]arvin-*.tar.gz
           mvn -q -Pdeveloper -pl developer -Ddeploydb
           mvn -q -Pdeveloper -pl developer -Ddeploydb-simulator
 

.github/workflows/codecov.yml

@@ -32,12 +32,12 @@ jobs:
     name: codecov
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
       - name: Set up JDK 17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: '17'

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: CodeQL Analysis
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+jobs:
+  codeql:
+    name: CodeQL
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ["actions"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "Security"

.github/workflows/docker-cloudstack-simulator.yml

@@ -47,7 +47,7 @@ jobs:
       - name: Set Docker repository name
         run: echo "DOCKER_REPOSITORY=apache" >> $GITHUB_ENV
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set ACS version
         run: echo "ACS_VERSION=$(grep '<version>' pom.xml | head -2 | tail -1 | cut -d'>' -f2 |cut -d'<' -f1)" >> $GITHUB_ENV

.github/workflows/license-templates/LICENSE.txt

@@ -0,0 +1,16 @@
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.

.github/workflows/main-sonar-check.yml

@@ -32,26 +32,26 @@ jobs:
     name: Main Sonar JaCoCo Build
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
       - name: Set up JDK17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: '17'
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
       - name: Cache local Maven repository
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}

.github/workflows/pre-commit.yml

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-name: Lint
+name: pre-commit
 
 on: [pull_request]
 
@@ -32,16 +32,18 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Check Out
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
       - name: Install
         run: |
           python -m pip install --upgrade pip
           pip install pre-commit
       - name: Set PY
         run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
-      - uses: actions/cache@v4
+      - uses: actions/cache@v5
         with:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
       - name: Run pre-commit
-        run: pre-commit run --all-files
+        run: pre-commit run --color=always --all-files
+      - name: Run manual pre-commit hooks
+        run: pre-commit run --color=always --all-files --hook-stage manual

.github/workflows/rat.yml

@@ -30,9 +30,9 @@ jobs:
   build:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up JDK 17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: '17'
           distribution: 'adopt'

.github/workflows/sonar-check.yml

@@ -33,27 +33,27 @@ jobs:
     name: Sonar JaCoCo Coverage
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: "refs/pull/${{ github.event.number }}/merge"
           fetch-depth: 0
 
       - name: Set up JDK17
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           distribution: 'temurin'
           java-version: '17'
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
       - name: Cache local Maven repository
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}

.github/workflows/stale.yml

@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v10
+        with:
+          stale-issue-message: 'This issue is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.'
+          stale-pr-message: 'This PR is stale because it has been open for 120 days with no activity. It may be removed by administrators of this project at any time. Remove the stale label or comment to request for removal of it to prevent this.'
+          close-issue-message: 'This issue was closed because it has been stale for 120 days with no activity.'
+          close-pr-message: 'This PR was closed because it has been stale for 240 days with no activity.'
+          stale-issue-label: 'no-issue-activity'
+          stale-pr-label: 'no-pr-activity'
+          days-before-stale: 120
+          days-before-close: -1
+          days-before-pr-close: 240
+          exempt-issue-labels: 'gsoc,good-first-issue,long-term-plan'
+          exempt-pr-labels: 'status:ready-for-merge,status:needs-testing,status:on-hold'

.github/workflows/ui.yml

@@ -31,10 +31,10 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set up Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v5
         with:
           node-version: 16
 
@@ -56,6 +56,7 @@ jobs:
           npm run test:unit
 
       - uses: codecov/codecov-action@v4
+        if: github.repository == 'apache/cloudstack'
         with:
           working-directory: ui
           files: ./coverage/lcov.info

.gitignore

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

.markdownlintignore

@@ -0,0 +1 @@
+CHANGES.md

.pre-commit-config.yaml

@@ -15,23 +15,111 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   # force all unspecified Python hooks to run python3
   python: python3
-minimum_pre_commit_version: "2.17.0"
+minimum_pre_commit_version: "3.2.0"
 repos:
   - repo: meta
     hooks:
       - id: identity
       - id: check-hooks-apply
+  - repo: https://github.com/thlorenz/doctoc.git
+    rev: v2.2.0
+    hooks:
+      - id: doctoc
+        name: Add TOC for Markdown files
+        files: ^CONTRIBUTING\.md$|^INSTALL\.md$|^README\.md$
+  - repo: https://github.com/oxipng/oxipng
+    rev: v9.1.5
+    hooks:
+      - id: oxipng
+        name: run oxipng
+        description: optimize PNG images with lossless compression
+        args: ['-o', '4', '--strip', 'safe', '--alpha']
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.27.2
+    hooks:
+      - id: gitleaks
+        name: run gitleaks
+        description: detect hardcoded secrets
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.5
+    hooks:
+      - id: chmod
+        name: set file permissions
+        args: ['644']
+        files: \.md$
+        stages: [manual]
+      - id: insert-license
+        name: add license for all Markdown files
+        files: \.md$
+        args:
+          - --comment-style
+          - '<!--|| -->'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+        exclude: ^(CHANGES|ISSUE_TEMPLATE|PULL_REQUEST_TEMPLATE)\.md$|^ui/docs/(full|smoke)-test-plan\.template\.md$
+      - id: insert-license
+        name: add license for all properties files
+        description: automatically adds a licence header to all properties files that don't have a license header
+        files: \.properties$
+        args:
+          - --comment-style
+          - '|#|'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: add license for all Shell files
+        description: automatically adds a licence header to all Shell files that don't have a license header
+        files: \.sh$
+        args:
+          - --comment-style
+          - '|#|'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: add license for all SQL files
+        files: \.sql$
+        args:
+          - --comment-style
+          - '|--|'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: add license for all Vue files
+        files: \.vue$
+        args:
+          - --comment-style
+          - '|//|'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
+      - id: insert-license
+        name: add license for all YAML files
+        description: automatically adds a licence header to all YAML files that don't have a license header
+        files: \.ya?ml$
+        args:
+          - --comment-style
+          - '|#|'
+          - --license-filepath
+          - .github/workflows/license-templates/LICENSE.txt
+          - --fuzzy-match-generates-todo
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v6.0.0
     hooks:
       #- id: check-added-large-files
       - id: check-case-conflict
       #- id: check-executables-have-shebangs
+      - id: check-illegal-windows-names
       - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+        files: \.sh$
       - id: check-symlinks
       - id: check-vcs-permalinks
       #- id: check-yaml
@@ -42,6 +130,7 @@ repos:
         exclude: >
           (?x)
           ^scripts/vm/systemvm/id_rsa\.cloud$|
+          ^server/src/test/java/org/apache/cloudstack/network/ssl/CertServiceTest\.java$|
           ^server/src/test/java/com/cloud/keystore/KeystoreTest\.java$|
           ^server/src/test/resources/certs/dsa_self_signed\.key$|
           ^server/src/test/resources/certs/non_root\.key$|
@@ -51,43 +140,35 @@ repos:
           ^server/src/test/resources/certs/rsa_self_signed\.key$|
           ^services/console-proxy/rdpconsole/src/test/doc/rdp-key\.pem$|
           ^systemvm/agent/certs/localhost\.key$|
-          ^systemvm/agent/certs/realhostip\.key$
+          ^systemvm/agent/certs/realhostip\.key$|
+          ^test/integration/smoke/test_ssl_offloading\.py$
       - id: end-of-file-fixer
-        exclude: \.vhd$
+        exclude: \.vhd$|\.svg$
-      #- id: fix-byte-order-marker
+      - id: file-contents-sorter
+        args: [--unique]
+        files: ^\.github/linters/codespell\.txt$
+      - id: fix-byte-order-marker
       - id: forbid-submodules
       - id: mixed-line-ending
-        exclude: \.(cs|xml)$
       - id: trailing-whitespace
-        files: \.(header|in|java|md|properties|py|rb|sh|sql|txt|vue|xml|yaml|yml)$
+        files: ^(LICENSE|NOTICE)$|\.(bat|cfg|cs|css|gitignore|header|in|install|java|md|properties|py|rb|rc|sh|sql|te|template|txt|ucls|vue|xml|xsl|yaml|yml)$|^cloud-cli/bindir/cloud-tool$|^debian/changelog$
         args: [--markdown-linebreak-ext=md]
         exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.4.1
     hooks:
       - id: codespell
         name: run codespell
         description: Check spelling with codespell
         args: [--ignore-words=.github/linters/codespell.txt]
-        exclude: ^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$
+        exclude: ^systemvm/agent/noVNC/|^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$|^server/src/test/java/org/apache/cloudstack/network/ssl/CertServiceTest.java$|^test/integration/smoke/test_ssl_offloading.py$
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
     - id: flake8
       args: [--config, .github/linters/.flake8]
-      exclude: >
-        (?x)
-        ^agent/bindir/cloud-setup-agent\.in$|
-        ^client/bindir/cloud-update-xenserver-licenses\.in$|
-        ^cloud-cli/bindir/cloud-tool$|
-        ^python/bindir/cloud-grab-dependent-library-versions$|
-        ^python/bindir/cloud-setup-baremetal$|
-        ^scripts/vm/hypervisor/xenserver/storagePlugin$|
-        ^scripts/vm/hypervisor/xenserver/vmopspremium$|
-        ^setup/bindir/cloud-setup-encryption\.in$|
-        ^venv/.*$
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.40.0
+    rev: v0.45.0
     hooks:
       - id: markdownlint
         name: run markdownlint
@@ -95,3 +176,13 @@ repos:
         args: [--config=.github/linters/.markdown-lint.yml]
         types: [markdown]
         files: \.(md|mdown|markdown)$
+  - repo: https://github.com/adrienverge/yamllint
+    rev: v1.37.1
+    hooks:
+      - id: yamllint
+        name: run yamllint
+        description: check YAML files with yamllint
+        args: [--config-file=.github/linters/.yamllint.yml]
+        types: [yaml]
+        files: \.ya?ml$
+        exclude: ^.*k8s-.*\.ya?ml$

CHANGES.md

@@ -242,7 +242,6 @@ Bug ID | Description
 [CLOUDSTACK-7722](https://issues.apache.org/jira/browse/CLOUDSTACK-7722) | add.label: Add button for tags show the label not "Add" text...
 [CLOUDSTACK-7246](https://issues.apache.org/jira/browse/CLOUDSTACK-7246) | VM deployment failed due to wrong in  script name createipalias.sh...
 
-
 Version 4.4.1
 -------------
 
@@ -276,7 +275,6 @@ Bug ID | Description
 [CLOUDSTACK-1632](https://issues.apache.org/jira/browse/CLOUDSTACK-1632) | Mistakes in authorizeSecurityGroup* API docs...
 [CLOUDSTACK-401](https://issues.apache.org/jira/browse/CLOUDSTACK-401)   | Storage options missing from table...
 
-
 Version 4.4.0
 -------------
 
@@ -646,12 +644,12 @@ Bug ID | Description
 Version 4.2.1
 -------------
 
-Release notes contain the list of [bug fixes](http://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.1/html/Release_Notes/version-4.2.html#issues-fixed-4.2.1)
+Release notes contain the list of [bug fixes](https://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.1/html/Release_Notes/version-4.2.html#issues-fixed-4.2.1)
 
 Version 4.2.0
 -------------
 Released on October 1 2013.
-Release notes contain the list of [bug fixes](http://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.0/html/Release_Notes/index.html)
+Release notes contain the list of [bug fixes](https://cloudstack.apache.org/docs/en-US/Apache_CloudStack/4.2.0/html/Release_Notes/index.html)
 
 Version 4.1.0
 -------------
@@ -930,7 +928,6 @@ Security Fixes:
 
  * CVE-2012-4501: Apache CloudStack configuration vulnerability
 
-
 Version 4.0.2
 ------------------------
 
@@ -979,7 +976,6 @@ Issues fixed in this release:
 * CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1.
 * CLOUDSTACK-2091: Error in API documentation for 4.0.x.
 
-
 Version 4.0.1-incubating
 ------------------------
 
@@ -1023,7 +1019,6 @@ Bugs fixed in this release:
 * CLOUDSTACK-961: Installation docs don't detail dependencies for building RPMs
 * CLOUDSTACK-995: Not able to add the KVM host
 
-
 Version 4.0.0-incubating
 ------------------------
 
@@ -1056,7 +1051,6 @@ Security Fixes:
 
  * CVE-2012-4501: Apache CloudStack configuration vulnerability
 
-
 Updating this file
 ------------------
 

CONTRIBUTING.md

@@ -1,46 +1,79 @@
-Contributing to Apache CloudStack (ACS)
+<!--
-=======================================
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Contributing to Apache CloudStack (ACS)
+
+## Summary
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [Summary](#summary)
+- [Bug fixes](#bug-fixes)
+- [Developing new features](#developing-new-features)
+- [PendingReleaseNotes file](#pendingreleasenotes-file)
+- [Fork the code](#fork-the-code)
+- [Making changes](#making-changes)
+- [Rebase `feature_x` to include updates from `upstream/main`](#rebase-feature_x-to-include-updates-from-upstreammain)
+- [Make a GitHub Pull Request to contribute your changes](#make-a-github-pull-request-to-contribute-your-changes)
+- [Cleaning up after a successful pull request](#cleaning-up-after-a-successful-pull-request)
+- [Release Principles](#release-principles)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Summary
 
-Summary
--------
 This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions.
-These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added.
+These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project, and you will submit a Pull Request for your changes to be added.
 
-_Lets get started!!!_
+_Let's get started!!!_
 
-Bug fixes
+## Bug fixes
----------
 
 It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches.
 Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch.
 This can be either the "current release" or the "previous release", depending on which ones are maintained.
 Since the goal is a stable main, bug fixes should be "merged forward" to the next branch in order: "previous release" -> "current release" -> main (in other words: old to new)
 
-Developing new features
+## Developing new features
------------------------
 
 Development should be done in a feature branch, branched off of main.
 Send a PR(steps below) to get it into main (2x LGTM applies).
 PR will only be merged when main is open, will be held otherwise until main is open again.
 No back porting / cherry-picking features to existing branches!
 
-PendingReleaseNotes file
+## PendingReleaseNotes file
-------------------------
+
-When developing a new feature or making a (major) change to a existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
+When developing a new feature or making a (major) change to an existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
 use this file as a source of information when compiling the Release Notes for a new release.
 
 When adding information to the PendingReleaseNotes file make sure that you write a good and understandable description of the new feature or change which you have developed.
 
 Updating the PendingReleaseNotes file is preferably a part of the original Pull Request, but that is up to the developers' discretion.
 
-Fork the code
+## Fork the code
--------------
 
 In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack)
 
-Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
+Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen, and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
 
-On your computer, follow these steps to setup a local repository for working on ACS:
+On your computer, follow these steps to set up a local repository for working on ACS:
 
 ```bash
 $ git clone https://github.com/YOUR_ACCOUNT/cloudstack.git
@@ -51,10 +84,7 @@ $ git fetch upstream
 $ git rebase upstream/main
 ```
 
-
+## Making changes
-Making changes
---------------
-
 
 It is important that you create a new branch to make changes on and that you do not change the `main` branch (other than to rebase in changes from `upstream/main`).  In this example I will assume you will be making your changes to a branch called `feature_x`.  This `feature_x` branch will be created on your local repository and will be pushed to your forked repository on GitHub.  Once this branch is on your fork you will create a Pull Request for the changes to be added to the ACS project.
 
@@ -70,9 +100,7 @@ $ git commit -a -m "descriptive commit message for your changes"
 
 > The `-b` specifies that you want to create a new branch called `feature_x`.  You only specify `-b` the first time you checkout because you are creating a new branch.  Once the `feature_x` branch exists, you can later switch to it with only `git checkout feature_x`.
 
-
+## Rebase `feature_x` to include updates from `upstream/main`
-Rebase `feature_x` to include updates from `upstream/main`
-------------------------------------------------------------
 
 It is important that you maintain an up-to-date `main` branch in your local repository.  This is done by rebasing in the code changes from `upstream/main` (the official ACS project repository) into your local repository.  You will want to do this before you start working on a feature as well as right before you submit your changes as a pull request.  I recommend you do this process periodically while you work to make sure you are working off the most recent project code.
 
@@ -92,13 +120,11 @@ $ git rebase main
 
 > Now your `feature_x` branch is up-to-date with all the code in `upstream/main`.
 
+## Make a GitHub Pull Request to contribute your changes
 
-Make a GitHub Pull Request to contribute your changes
+When you are happy with your changes, and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
------------------------------------------------------
 
-When you are happy with your changes and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
+Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. In case of UI PRs, a screenshot is preferred.
-
-Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. Incase of UI PRs, a screenshot is preferred.
 
 > **IMPORTANT:** Make sure you have rebased your `feature_x` branch to include the latest code from `upstream/main` _before_ you do this.
 
@@ -107,7 +133,7 @@ $ git push origin main
 $ git push origin feature_x
 ```
 
-Now that the `feature_x` branch has been pushed to your GitHub repository, you can initiate the pull request.  
+Now that the `feature_x` branch has been pushed to your GitHub repository, you can initiate the pull request.
 
 To initiate the pull request, do the following:
 
@@ -118,9 +144,7 @@ To initiate the pull request, do the following:
 
 If you are requested to make modifications to your proposed changes, make the changes locally on your `feature_x` branch, re-push the `feature_x` branch to your fork.  The existing pull request should automatically pick up the change and update accordingly.
 
-
+## Cleaning up after a successful pull request
-Cleaning up after a successful pull request
--------------------------------------------
 
 Once the `feature_x` branch has been committed into the `upstream/main` branch, your local `feature_x` branch and the `origin/feature_x` branch are no longer needed.  If you want to make additional changes, restart the process with a new branch.
 
@@ -134,6 +158,6 @@ $ git branch -D feature_x
 $ git push origin :feature_x
 ```
 
-Release Principles
+## Release Principles
-------------------
+
 Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up

INSTALL.md

@@ -1,15 +1,46 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
 # Apache CloudStack Installation basics
 
 This document describes how to develop, build, package and install Apache
-CloudStack. For more information please refer to the official [documentation](http://docs.cloudstack.apache.org)
+CloudStack. For more information please refer to the official [documentation](https://docs.cloudstack.apache.org)
 or the developer [wiki](https://cwiki.apache.org/confluence/display/CLOUDSTACK/Home).
 
 Apache CloudStack developers use various platforms for development, this guide
 was tested against a CentOS 7 x86_64 setup.
 
-* [Setting up development environment](https://cwiki.apache.org/confluence/display/CLOUDSTACK/Setting+up+CloudStack+Development+Environment) for Apache CloudStack.
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
-* [Building](https://cwiki.apache.org/confluence/display/CLOUDSTACK/How+to+build+CloudStack) Apache CloudStack.
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
-* [Appliance based development](https://github.com/rhtyd/monkeybox)
+
+- [Setting up Development Environment](#setting-up-development-environment)
+  - [Using jenv and/or pyenv for Version Management](#using-jenv-andor-pyenv-for-version-management)
+- [Getting the Source Code](#getting-the-source-code)
+- [Building](#building)
+- [To bring up CloudStack UI](#to-bring-up-cloudstack-ui)
+- [Building with non-redistributable plugins](#building-with-non-redistributable-plugins)
+- [Packaging and Installation](#packaging-and-installation)
+  - [Debian/Ubuntu](#debianubuntu)
+  - [RHEL/CentOS](#rhelcentos)
+- [Notes](#notes)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
 ## Setting up Development Environment
 
@@ -18,29 +49,30 @@ Install tools and dependencies used for development:
     # yum -y install git java-17-openjdk java-17-openjdk-devel \
       mysql mysql-server mkisofs git gcc python MySQL-python openssh-clients wget
 
-Set up Maven (3.6.0):
+Set up Maven (3.9.10):
 
-    # wget http://www.us.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
+    # wget https://dlcdn.apache.org/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.tar.gz
-    # tar -zxvf apache-maven-3.6.3-bin.tar.gz -C /usr/local
+    # sudo tar -zxvf apache-maven-3.9.10-bin.tar.gz -C /usr/local
     # cd /usr/local
-    # ln -s apache-maven-3.6.3 maven
+    # sudo ln -s apache-maven-3.9.10 maven
     # echo export M2_HOME=/usr/local/maven >> ~/.bashrc # or .zshrc or .profile
     # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile
     # source ~/.bashrc
 
-Setup up NodeJS (LTS):
+Setup up Node.js 16:
 
-    # curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash -
+    # curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash -
     # sudo yum install nodejs
     # sudo npm install -g @vue/cli npm-check-updates
 
 Start the MySQL service:
 
     $ service mysqld start
+    $ mysql_secure_installation
 
 ### Using jenv and/or pyenv for Version Management
 
-CloudStack is built using Java and Python.  To make selection of these tools versions more consistent and ease installation for developers, optional support for [jenv](http://www.jenv.be/) and [pyenv](https://github.com/yyuu/pyenv) with [virtualenv]|(https://github.com/yyuu/pyenv-virtualenv) is provided.  jenv installation instructions are available here and pyenv installation instructions are available here.  For users of [oh-my-zsh](http://ohmyz.sh/) there is a pyenv plugin available to trigger configuration of pyenv in a shell session.
+CloudStack is built using Java and Python.  To make selection of these tools versions more consistent and ease installation for developers, optional support for [jenv](http://www.jenv.be/) and [pyenv](https://github.com/yyuu/pyenv) with [virtualenv]|(https://github.com/yyuu/pyenv-virtualenv) is provided.  jenv installation instructions are available here and pyenv installation instructions are available here.  For users of [oh-my-zsh](https://ohmyz.sh/) there is a pyenv plugin available to trigger configuration of pyenv in a shell session.
 
 Following installation, execute the following commands to configure jenv and pyenv for use with CloudStack development:
 
@@ -86,13 +118,33 @@ Start the management server:
 
 If this works, you've successfully setup a single server Apache CloudStack installation.
 
-Open the following URL on your browser to access the Management Server UI:
+To access the Management Server UI, follow the following procedure:
-
-    http://localhost:8080/client/
 
 The default credentials are; user: admin, password: password and the domain
 field should be left blank which is defaulted to the ROOT domain.
 
+## To bring up CloudStack UI
+
+Move to UI Directory
+
+    $ cd /path/to/cloudstack/ui
+
+To install dependencies.
+
+    $ npm install
+
+To build the project.
+
+    $ npm run build
+
+For Development Mode.
+
+    $ npm start
+
+Make sure to set `CS_URL=http://localhost:8080` on the `.env.local` file on UI.
+
+You should be able to run the management server on http://localhost:5050
+
 ## Building with non-redistributable plugins
 
 CloudStack supports several plugins that depend on libraries with distribution restrictions.
@@ -150,7 +202,7 @@ All the rpm packages will be created in `dist/rpmbuild/RPMS/x86_64` directory.
 
 ## Notes
 
-If you will be using Xen as your hypervisor, please download [vhd-util](http://download.cloudstack.org/tools/vhd-util)
+If you will be using Xen as your hypervisor, please download [vhd-util](https://download.cloudstack.org/tools/vhd-util)
 
 If management server is installed on RHEL/CentOS, then copy vhd-util into:
 

ISSUE_TEMPLATE.md

@@ -35,17 +35,14 @@ New line separated list of affected versions, commit ID for issues on main branc
 Information about the configuration if relevant, e.g. basic network, advanced networking, etc.  N/A otherwise
 -->
 
-
 ##### OS / ENVIRONMENT
 <!--
 Information about the environment if relevant, N/A otherwise
 -->
 
-
 ##### SUMMARY
 <!-- Explain the problem/feature briefly -->
 
-
 ##### STEPS TO REPRODUCE
 <!--
 For bugs, show exactly how to reproduce the problem, using a minimal test-case. Use Screenshots if accurate.

LICENSE

@@ -177,14 +177,14 @@ Copyright (c) 2014 The Apache Software Foundation
       of your accepting any such warranty or additional liability.
 
    END OF TERMS AND CONDITIONS
-            
+
 
 This distribution contains third party resources.
 Within the console-proxy/js directory
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
-            Copyright (c) 2009, John Resig 
+            Copyright (c) 2009, John Resig
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -192,10 +192,10 @@ Within the console-proxy/js directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -203,43 +203,43 @@ Within the console-proxy/js directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from John Resig   
+        from John Resig
-            jquery.js 
+            jquery.js
 
 
 Within the systemvm/debian/etc directory
     placed in the public domain
-        by Adiscon GmbH  http://www.adiscon.com/ 
+        by Adiscon GmbH  http://www.adiscon.com/
-            rsyslog.conf 
+            rsyslog.conf
-        by Simon Kelley   
+        by Simon Kelley
-            dnsmasq.conf 
+            dnsmasq.conf
-            vpcdnsmasq.conf 
+            vpcdnsmasq.conf
 
 Within the systemvm/debian/etc/apache2 directory
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2012 The Apache Software Foundation
-        from The Apache Software Foundation  http://www.apache.org/ 
+        from The Apache Software Foundation  http://www.apache.org/
-            httpd.conf 
+            httpd.conf
             vhost.template
 
 Within the systemvm/debian/etc/ssh/ directory
     licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause  (as follows)
- 
+
-            
+
             Redistribution and use in source and binary forms, with or without modification,
             are permitted provided that the following conditions are met:
-            
+
             Redistributions of source code must retain the above copyright notice, this list
             of  conditions and the following disclaimer. Redistributions in binary form must
             reproduce the above copyright notice, this list  of conditions and the following
             disclaimer in the documentation and/or other materials  provided with the
             distribution.
-            
+
             Neither the name of the author nor the names of contributors may be used to
             endorse  or promote products derived from this software without specific prior
             written permission.
-            
+
             THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
             ANY  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
             WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -250,55 +250,55 @@ Within the systemvm/debian/etc/ssh/ directory
             ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
             (INCLUDING  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
             SOFTWARE, EVEN IF ADVISED  OF THE POSSIBILITY OF SUCH DAMAGE.
-                        
+
-        from OpenSSH Project  http://www.openssh.org/ 
+        from OpenSSH Project  http://www.openssh.org/
-            sshd_config 
+            sshd_config
 
 Within the systemvm/debian/root/redundant_router directory
     placed in the public domain
-        by The netfilter.org project  http://www.netfilter.org/ 
+        by The netfilter.org project  http://www.netfilter.org/
-            conntrackd.conf.templ 
+            conntrackd.conf.templ
 
 Within the scripts/storage/secondary directory
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2010-2011 OpenStack, LLC.
-        from OpenStack, LLC  http://www.openstack.org 
+        from OpenStack, LLC  http://www.openstack.org
-            swift 
+            swift
 
 Within the scripts/vm/hypervisor/xenserver directory
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2010-2011 OpenStack, LLC.
-        from OpenStack, LLC  http://www.openstack.org 
+        from OpenStack, LLC  http://www.openstack.org
-            swift 
+            swift
 
 Within the ui/lib directory
     placed in the public domain
-        by Eric Meyer  http://meyerweb.com/eric/ 
+        by Eric Meyer  http://meyerweb.com/eric/
             reset.css  from http://meyerweb.com/eric/tools/css/reset/
 
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2006 Google Inc.
-        from Google Inc.  http://google.com 
+        from Google Inc.  http://google.com
             excanvas.js  from http://code.google.com/p/explorercanvas/
 
     licensed under the BSD (2-clause) http://www.opensource.org/licenses/BSD-2-Clause  (as follows)
 
             Copyright (c) 2008 George McGinley Smith
-            All rights reserved. 
+            All rights reserved.
-            
+
             Redistribution and use in source and binary forms, with or without modification,
             are permitted provided that the following conditions are met:
-            
+
             Redistributions of source code must retain the above copyright notice, this list
             of  conditions and the following disclaimer. Redistributions in binary form must
             reproduce the above copyright notice, this list  of conditions and the following
             disclaimer in the documentation and/or other materials  provided with the
             distribution.
-            
+
             Neither the name of the author nor the names of contributors may be used to
             endorse  or promote products derived from this software without specific prior
             written permission.
-            
+
             THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
             ANY  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
             WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -309,13 +309,13 @@ Within the ui/lib directory
             ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
             (INCLUDING  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
             SOFTWARE, EVEN IF ADVISED  OF THE POSSIBILITY OF SUCH DAMAGE.
-                        
+
-        from George McGinley Smith   
+        from George McGinley Smith
-            jquery.easing.js 
+            jquery.easing.js
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
- 
+
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -323,10 +323,10 @@ Within the ui/lib directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -334,14 +334,14 @@ Within the ui/lib directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from The Dojo Foundation  http://dojofoundation.org/ 
+        from The Dojo Foundation  http://dojofoundation.org/
             require.js  from http://github.com/jrburke/requirejs
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
-            Copyright (c) 2011, John Resig 
+            Copyright (c) 2011, John Resig
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -349,10 +349,10 @@ Within the ui/lib directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -360,14 +360,14 @@ Within the ui/lib directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from John Resig   
+        from John Resig
-            jquery.js 
+            jquery.js
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
             Copyright (c) 2014 Jörn Zaefferer
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -375,10 +375,10 @@ Within the ui/lib directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -386,9 +386,9 @@ Within the ui/lib directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from Jorn Zaefferer   
+        from Jorn Zaefferer
-            jquery.validate.js 
+            jquery.validate.js
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
@@ -418,8 +418,8 @@ Within the ui/lib directory
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
-            Copyright (c) 2010, Sebastian Tschan 
+            Copyright (c) 2010, Sebastian Tschan
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -427,10 +427,10 @@ Within the ui/lib directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -438,14 +438,14 @@ Within the ui/lib directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from Sebastian Tschan  https://blueimp.net 
+        from Sebastian Tschan  https://blueimp.net
-            jquery.md5.js 
+            jquery.md5.js
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
-            Copyright (c) 2006 Klaus Hartl (stilbuero.de) 
+            Copyright (c) 2006 Klaus Hartl (stilbuero.de)
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -453,10 +453,10 @@ Within the ui/lib directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -464,15 +464,15 @@ Within the ui/lib directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from Klaus Hartl  http://stilbuero.de 
+        from Klaus Hartl  http://stilbuero.de
-            jquery.cookies.js 
+            jquery.cookies.js
 
 Within the ui/lib/flot directory
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
-            Released under the MIT license by IOLA, December 2007. 
+            Released under the MIT license by IOLA, December 2007.
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -480,10 +480,10 @@ Within the ui/lib/flot directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -491,24 +491,24 @@ Within the ui/lib/flot directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from IOLA  http://www.iola.dk/ 
+        from IOLA  http://www.iola.dk/
-            jquery.flot.crosshair.js 
+            jquery.flot.crosshair.js
-            jquery.flot.fillbetween.js 
+            jquery.flot.fillbetween.js
-            jquery.flot.image.js 
+            jquery.flot.image.js
-            jquery.flot.js 
+            jquery.flot.js
-            jquery.flot.navigate.js 
+            jquery.flot.navigate.js
-            jquery.flot.resize.js 
+            jquery.flot.resize.js
-            jquery.flot.selection.js 
+            jquery.flot.selection.js
-            jquery.flot.stack.js 
+            jquery.flot.stack.js
-            jquery.flot.symbol.js 
+            jquery.flot.symbol.js
-            jquery.flot.threshold.js 
+            jquery.flot.threshold.js
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
             Created by Brian Medendorp, June 2009
-            Updated November 2009 with contributions from: btburnett3, Anthony Aragues and Xavi Ivars 
+            Updated November 2009 with contributions from: btburnett3, Anthony Aragues and Xavi Ivars
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -516,10 +516,10 @@ Within the ui/lib/flot directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -527,13 +527,13 @@ Within the ui/lib/flot directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from Brian Medendorp   
+        from Brian Medendorp
-            jquery.pie.js 
+            jquery.pie.js
 
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
- 
+
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -541,10 +541,10 @@ Within the ui/lib/flot directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -552,14 +552,14 @@ Within the ui/lib/flot directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from Ole Laursen   
+        from Ole Laursen
-            jquery.colorhelpers.js 
+            jquery.colorhelpers.js
 
 Within the ui/lib/jquery-ui directory
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
- 
+
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -567,10 +567,10 @@ Within the ui/lib/jquery-ui directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -578,17 +578,17 @@ Within the ui/lib/jquery-ui directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from jQuery UI Developers  http://jqueryui.com/about 
+        from jQuery UI Developers  http://jqueryui.com/about
-            css/jquery-ui.css 
+            css/jquery-ui.css
-            index.html 
+            index.html
-            js/jquery-ui.js 
+            js/jquery-ui.js
 
 Within the ui/lib/qunit directory
     licensed under the MIT License http://www.opensource.org/licenses/mit-license.php  (as follows)
 
-            Copyright (c) 2012 John Resig, Jörn Zaefferer 
+            Copyright (c) 2012 John Resig, Jörn Zaefferer
-            
+
             Permission is hereby granted, free  of charge, to any person obtaining
             a  copy  of this  software  and  associated  documentation files  (the
             "Software"), to  deal in  the Software without  restriction, including
@@ -596,10 +596,10 @@ Within the ui/lib/qunit directory
             distribute,  sublicense, and/or sell  copies of  the Software,  and to
             permit persons to whom the Software  is furnished to do so, subject to
             the following conditions:
-            
+
             The  above  copyright  notice  and  this permission  notice  shall  be
             included in all copies or substantial portions of the Software.
-            
+
             THE  SOFTWARE IS  PROVIDED  "AS  IS", WITHOUT  WARRANTY  OF ANY  KIND,
             EXPRESS OR  IMPLIED, INCLUDING  BUT NOT LIMITED  TO THE  WARRANTIES OF
             MERCHANTABILITY,    FITNESS    FOR    A   PARTICULAR    PURPOSE    AND
@@ -607,20 +607,20 @@ Within the ui/lib/qunit directory
             LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
             OF CONTRACT, TORT OR OTHERWISE,  ARISING FROM, OUT OF OR IN CONNECTION
             WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-                        
+
-        from Jorn Zaefferer   
+        from Jorn Zaefferer
             qunit.css  from http://docs.jquery.com/QUnit
             qunit.js  from http://docs.jquery.com/QUnit
 
 Within the utils/src/main/java/com/cloud/utils/db directory
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2004 Clinton Begin
-        from Clinton Begin  http://code.google.com/p/mybatis/ 
+        from Clinton Begin  http://code.google.com/p/mybatis/
             ScriptRunner.java  from http://code.google.com/p/mybatis/
 
 Within the utils/src/main/java/org/apache/commons/httpclient/contrib/ssl directory
     licensed under the Apache License, Version 2 http://www.apache.org/licenses/LICENSE-2.0.txt  (as above)
     Copyright (c) 2007 The Apache Software Foundation
-        from The Apache Software Foundation  http://www.apache.org/ 
+        from The Apache Software Foundation  http://www.apache.org/
-            EasySSLProtocolSocketFactory.java 
+            EasySSLProtocolSocketFactory.java
-            EasyX509TrustManager.java 
+            EasyX509TrustManager.java

NOTICE

@@ -1,62 +1,62 @@
 Apache CloudStack
 Copyright 2014 The Apache Software Foundation
-  
+
   This product includes software developed at
   The Apache Software Foundation (http://www.apache.org/).
-      
 
 
-  
+
+
   This distribution contains third party resources requiring the following notices:
 
-    
+
-  For 
+  For
     jquery.js
-  
+
 
         jQuery JavaScript Library v1.3.2
         http://jquery.com/
-        
+
         Copyright (c) 2009 John Resig
         Dual licensed under the MIT and GPL licenses.
         http://docs.jquery.com/License
-        
+
         Date: 2009-02-19 17:34:21 -0500 (Thu, 19 Feb 2009)
         Revision: 6246
 
-    
+
-  For 
+  For
     jquery.js
-  
+
 
         jQuery JavaScript Library v1.6.4
         http://jquery.com/
-        
+
         Copyright 2011, John Resig
         Dual licensed under the MIT or GPL Version 2 licenses.
         http://jquery.org/license
-        
+
         Includes Sizzle.js
         http://sizzlejs.com/
         Copyright 2011, The Dojo Foundation
         Released under the MIT, BSD, and GPL Licenses.
-        
+
         Date: Mon Sep 12 18:54:48 2011 -0400
 
-    
+
-  For 
+  For
     jquery.md5.js
-  
+
 
         jQuery MD5 Plugin 1.2.1
         https://github.com/blueimp/jQuery-MD5
-        
+
         Copyright 2010, Sebastian Tschan
         https://blueimp.net
-        
+
         Licensed under the MIT license:
         http://creativecommons.org/licenses/MIT/
-        
+
         Based on
         A JavaScript implementation of the RSA Data Security, Inc. MD5 Message
         Digest Algorithm, as defined in RFC 1321.
@@ -65,15 +65,15 @@ Copyright 2014 The Apache Software Foundation
         Distributed under the BSD License
         See http://pajhome.org.uk/crypt/md5 for more info.
 
-    
+
-  For 
+  For
     jquery.colorhelpers.js
-  
+
 
         Plugin for jQuery for working with colors.
-        
+
         Version 1.1.
-        
+
         Inspiration from jQuery color animation plugin by John Resig.
-        
+
         Released under the MIT license by Ole Laursen, October 2009.

PRE_COMMIT.md

@@ -0,0 +1,62 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# pre-commit
+
+We run [pre-commit](https://pre-commit.com/) with
+[GitHub Actions](https://github.com/apache/cloudstack/blob/main/.github/workflows/pre-commit.yml) so installation on your
+local machine is currently optional.
+
+The `pre-commit` [configuration file](https://github.com/apache/cloudstack/blob/main/.pre-commit-config.yaml)
+is in the repository root. Before you can run the hooks, you need to have `pre-commit` installed. `pre-commit` is a
+[Python package](https://pypi.org/project/pre-commit/).
+
+From the repository root run: `pip install -r requirements-dev.txt` to install `pre-commit` and after you install
+`pre-commit` you will then need to install the pre-commit hooks by running `pre-commit install`.
+
+The hooks run when running `git commit` and also from the command line with `pre-commit`. Some of the hooks will auto
+fix the code after the hooks fail whilst most will print error messages from the linters. If a hook fails the overall
+commit will fail, and you will need to fix the issues or problems and `git add` and `git commit` again. On `git commit`
+the hooks will run mostly only against modified files so if you want to test all hooks against all files and when you
+are adding a new hook you should always run:
+
+`pre-commit run --all-files`
+
+Sometimes you might need to skip a hook to commit because the hook is stopping you from committing or your computer
+might not have all the installation requirements for all the hooks. The `SKIP` variable is comma separated for two or
+more hooks:
+
+`SKIP=codespell git commit -m "foo"`
+
+The same applies when running pre-commit:
+
+`SKIP=codespell pre-commit run --all-files`
+
+Occasionally you can have more serious problems when using `pre-commit` with `git commit`. You can use `--no-verify` to
+commit and stop `pre-commit` from checking the hooks. For example:
+
+`git commit --no-verify -m "foo"`
+
+If you are having major problems using `pre-commit` you can always uninstall it.
+
+To run a single hook use `pre-commit run --all-files <hook_id>`
+
+For example just run the `codespell` hook:
+
+`pre-commit run --all-files codespell`

PULL_REQUEST_TEMPLATE.md

@@ -22,8 +22,8 @@ This PR...
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] Enhancement (improves an existing feature and functionality)
 - [ ] Cleanup (Code refactoring and cleanup, that may add test cases)
-- [ ] build/CI
+- [ ] Build/CI
-- [ ] test (unit or integration test code)
+- [ ] Test (unit or integration test code)
 
 ### Feature/Enhancement Scale or Bug Severity
 
@@ -40,10 +40,8 @@ This PR...
 - [ ] Minor
 - [ ] Trivial
 
-
 ### Screenshots (if appropriate):
 
-
 ### How Has This Been Tested?
 
 <!-- Please describe in detail how you tested your changes. -->
@@ -53,5 +51,4 @@ This PR...
 
 <!-- see how your change affects other areas of the code, etc. -->
 
-
 <!-- Please read the [CONTRIBUTING](https://github.com/apache/cloudstack/blob/main/CONTRIBUTING.md) document -->

README.md

@@ -1,7 +1,54 @@
-# Apache CloudStack [![Build Status](https://github.com/apache/cloudstack/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/build.yml) [![UI Build](https://github.com/apache/cloudstack/actions/workflows/ui.yml/badge.svg)](https://github.com/apache/cloudstack/actions/workflows/ui.yml) [![License Check](https://github.com/apache/cloudstack/actions/workflows/rat.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/rat.yml) [![Simulator CI](https://github.com/apache/cloudstack/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/ci.yml) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache_cloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache_cloudstack) [![codecov](https://codecov.io/gh/apache/cloudstack/branch/main/graph/badge.svg)](https://codecov.io/gh/apache/cloudstack)
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Apache CloudStack
+
+[![Build Status](https://github.com/apache/cloudstack/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/build.yml)
+[![codecov](https://codecov.io/gh/apache/cloudstack/branch/main/graph/badge.svg)](https://codecov.io/gh/apache/cloudstack)
+[![Docker CloudStack Simulator Status](https://github.com/apache/cloudstack/actions/workflows/docker-cloudstack-simulator.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/docker-cloudstack-simulator.yml)
+[![License Check](https://github.com/apache/cloudstack/actions/workflows/rat.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/rat.yml)
+[![Linter Status](https://github.com/apache/cloudstack/actions/workflows/linter.yml/badge.svg)](https://github.com/apache/cloudstack/actions/workflows/linter.yml)
+[![Merge Conflict Checker Status](https://github.com/apache/cloudstack/actions/workflows/merge-conflict-checker.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/merge-conflict-checker.yml)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=apache_cloudstack&metric=alert_status)](https://sonarcloud.io/dashboard?id=apache_cloudstack)
+[![Simulator CI](https://github.com/apache/cloudstack/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/ci.yml)
+[![UI Build](https://github.com/apache/cloudstack/actions/workflows/ui.yml/badge.svg?branch=main)](https://github.com/apache/cloudstack/actions/workflows/ui.yml)
 
 [![Apache CloudStack](tools/logo/apache_cloudstack.png)](https://cloudstack.apache.org/)
 
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [Who Uses CloudStack?](#who-uses-cloudstack)
+- [Demo](#demo)
+- [Getting Started](#getting-started)
+- [Getting Source Repository](#getting-source-repository)
+- [Documentation](#documentation)
+- [News and Events](#news-and-events)
+- [Getting Involved and Contributing](#getting-involved-and-contributing)
+- [Reporting Security Vulnerabilities](#reporting-security-vulnerabilities)
+- [License](#license)
+- [Notice of Cryptographic Software](#notice-of-cryptographic-software)
+- [Star History](#star-history)
+- [Contributors](#contributors)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
 Apache CloudStack is open source software designed to deploy and manage large
 networks of virtual machines, as a highly available, highly scalable
 Infrastructure as a Service (IaaS) cloud computing platform. CloudStack is used
@@ -21,7 +68,7 @@ OVM and LXC containers.
 Users can manage their cloud with an easy to use Web interface, command line
 tools, and/or a full-featured query based API.
 
-For more information on Apache CloudStack, please visit the [website](http://cloudstack.apache.org)
+For more information on Apache CloudStack, please visit the [website](https://cloudstack.apache.org)
 
 ## Who Uses CloudStack?
 
@@ -78,10 +125,10 @@ via GitHub pull requests.
 ## Getting Involved and Contributing
 
 Interested in helping out with Apache CloudStack? Great! We welcome
-participation from anybody willing to work [The Apache Way](http://theapacheway.com) and make a
+participation from anybody willing to work [The Apache Way](https://theapacheway.com) and make a
 contribution. Note that you do not have to be a developer in order to contribute
 to Apache CloudStack. We need folks to help with documentation, translation,
-promotion etc. See our contribution [page](http://cloudstack.apache.org/contribute.html).
+promotion etc. See our contribution [page](https://cloudstack.apache.org/contribute.html).
 
 If you are a frequent contributors, you can request to be added as collaborators
 (see https://cwiki.apache.org/confluence/display/INFRA/Git+-+.asf.yaml+features#Git.asf.yamlfeatures-AssigningexternalcollaboratorswiththetriageroleonGitHub)
@@ -92,7 +139,7 @@ You may do so by sharing your GitHub users ID or raise a GitHub issue.
 
 If you're interested in learning more or participating in the Apache CloudStack
 project, the mailing lists are the best way to do that. While the project has
-several communications channels, the [mailing lists](http://cloudstack.apache.org/mailing-lists.html) are the most active and the
+several communications channels, the [mailing lists](https://cloudstack.apache.org/mailing-lists.html) are the most active and the
 official channels for making decisions about the project itself.
 
 Mailing lists:
@@ -112,7 +159,7 @@ released version of CloudStack, please report it to `security@apache.org` with
 details about the vulnerability, how it might be exploited, and any additional
 information that might be useful.
 
-For more details, please visit our security [page](http://cloudstack.apache.org/security.html).
+For more details, please visit our security [page](https://cloudstack.apache.org/security.html).
 
 ## License
 
@@ -160,3 +207,11 @@ The following provides more details on the included cryptographic software:
 * CloudStack makes use of the Bouncy Castle general-purpose encryption library.
 * CloudStack can optionally interact with and control OpenSwan-based VPNs.
 * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation.
+
+## Star History
+
+[![Apache CloudStack Star History](https://api.star-history.com/svg?repos=apache/cloudstack&type=Date)](https://www.star-history.com/#apache/cloudstack&Date)
+
+## Contributors
+
+[![Apache CloudStack Contributors](https://contrib.rocks/image?repo=apache/cloudstack&anon=0&max=500)](https://github.com/apache/cloudstack/graphs/contributors)

agent/bindir/cloud-setup-agent.in

@@ -35,7 +35,7 @@ for pythonpath in (
 
 from cloudutils.cloudException import CloudRuntimeException, CloudInternalException
 from cloudutils.utilities import initLoging, bash
-from cloudutils.configFileOps import  configFileOps
+from cloudutils.configFileOps import configFileOps
 from cloudutils.globalEnv import globalEnv
 from cloudutils.networkConfig import networkConfig
 from cloudutils.syscfg import sysConfigFactory
@@ -43,35 +43,41 @@ from cloudutils.serviceConfig import configureLibvirtConfig, configure_libvirt_t
 
 from optparse import OptionParser
 
+
 def getUserInputs():
     print("Welcome to the CloudStack Agent Setup:")
 
     cfo = configFileOps("@AGENTSYSCONFDIR@/agent.properties")
     oldMgt = cfo.getEntry("host")
 
-    mgtSvr = input("Please input the Management Server Hostname/IP-Address:[%s]"%oldMgt)
+    mgtSvr = input(
+        "Please input the Management Server Hostname/IP-Address:[%s]" % oldMgt
+    )
     if mgtSvr == "":
         mgtSvr = oldMgt
     try:
         socket.getaddrinfo(mgtSvr, 443)
     except:
-        print("Failed to resolve %s. Please input a valid hostname or IP-Address."%mgtSvr)
+        print(
+            "Failed to resolve %s. Please input a valid hostname or IP-Address."
+            % mgtSvr
+        )
         exit(1)
 
     oldToken = cfo.getEntry("zone")
-    zoneToken = input("Please input the Zone Id:[%s]"%oldToken)
+    zoneToken = input("Please input the Zone Id:[%s]" % oldToken)
 
     if zoneToken == "":
         zoneToken = oldToken
 
     oldPod = cfo.getEntry("pod")
-    podId = input("Please input the Pod Id:[%s]"%oldPod)
+    podId = input("Please input the Pod Id:[%s]" % oldPod)
 
     if podId == "":
-       podId  = oldToken
+        podId = oldToken
 
     oldCluster = cfo.getEntry("cluster")
-    clusterId = input("Please input the Cluster Id:[%s]"%oldCluster)
+    clusterId = input("Please input the Cluster Id:[%s]" % oldCluster)
     if clusterId == "":
         clusterId = oldCluster
 
@@ -79,18 +85,20 @@ def getUserInputs():
     if oldHypervisor == "":
         oldHypervisor = "kvm"
 
-    hypervisor = input("Please input the Hypervisor type kvm/lxc:[%s]"%oldHypervisor)
+    hypervisor = input("Please input the Hypervisor type kvm/lxc:[%s]" % oldHypervisor)
     if hypervisor == "":
         hypervisor = oldHypervisor
 
     try:
         defaultNic = networkConfig.getDefaultNetwork()
     except:
-        print("Failed to get default route. Please configure your network to have a default route")
+        print(
+            "Failed to get default route. Please configure your network to have a default route"
+        )
         exit(1)
 
     defNic = defaultNic.name
-    network = input("Please choose which network used to create VM:[%s]"%defNic)
+    network = input("Please choose which network used to create VM:[%s]" % defNic)
     if network == "":
         if defNic == "":
             print("You need to specify one of Nic or bridge on your system")
@@ -100,7 +108,8 @@ def getUserInputs():
 
     return [mgtSvr, zoneToken, network, podId, clusterId, hypervisor]
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     initLoging("@AGENTLOGDIR@/setup.log")
     glbEnv = globalEnv()
 
@@ -108,13 +117,23 @@ if __name__ == '__main__':
     glbEnv.agentMode = "Agent"
     parser = OptionParser()
     parser.add_option("-a", action="store_true", dest="auto", help="auto mode")
-    parser.add_option("-m", "--host", dest="mgt", help="Management server hostname or IP-Address")
+    parser.add_option(
+        "-m", "--host", dest="mgt", help="Management server hostname or IP-Address"
+    )
     parser.add_option("-z", "--zone", dest="zone", help="zone id")
     parser.add_option("-p", "--pod", dest="pod", help="pod id")
     parser.add_option("-c", "--cluster", dest="cluster", help="cluster id")
-    parser.add_option("-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type")
+    parser.add_option(
+        "-t", "--hypervisor", default="kvm", dest="hypervisor", help="hypervisor type"
+    )
     parser.add_option("-g", "--guid", dest="guid", help="guid")
-    parser.add_option("-s", action="store_true", default=False, dest="secure", help="Secure and enable TLS for libvirtd")
+    parser.add_option(
+        "-s",
+        action="store_true",
+        default=False,
+        dest="secure",
+        help="Secure and enable TLS for libvirtd",
+    )
     parser.add_option("--pubNic", dest="pubNic", help="Public traffic interface")
     parser.add_option("--prvNic", dest="prvNic", help="Private traffic interface")
     parser.add_option("--guestNic", dest="guestNic", help="Guest traffic interface")
@@ -140,15 +159,15 @@ if __name__ == '__main__':
         glbEnv.pod = userInputs[3]
         glbEnv.cluster = userInputs[4]
         glbEnv.hypervisor = userInputs[5]
-        #generate UUID
+        # generate UUID
         glbEnv.uuid = old_config.getEntry("guid")
         if glbEnv.uuid == "":
             glbEnv.uuid = bash("uuidgen").getStdout()
     else:
         for para, value in list(options.__dict__.items()):
             if value is None:
-                print("Missing operand:%s"%para)
+                print("Missing operand:%s" % para)
-                print("Try %s --help for more information"%sys.argv[0])
+                print("Try %s --help for more information" % sys.argv[0])
                 sys.exit(1)
 
         glbEnv.uuid = options.guid
@@ -168,7 +187,7 @@ if __name__ == '__main__':
     try:
         syscfg.config()
         print("CloudStack Agent setup is done!")
-    except (CloudRuntimeException,CloudInternalException) as e:
+    except (CloudRuntimeException, CloudInternalException) as e:
         print(e)
         print("Try to restore your system:")
         try:

agent/conf/agent.properties

@@ -209,12 +209,13 @@ hypervisor.type=kvm
 # the management server would send.
 # In case of arm64 (aarch64), this will change the machine type to 'virt' and
 # adds a SCSI and a USB controller in the domain xml.
-# Possible values: x86_64 | aarch64
+# Possible values: x86_64 | aarch64 | s390x
 # If null (default), defaults to the VM's OS architecture
 #guest.cpu.arch=
 
-# This param will require CPU features on the CPU section.
+# Specifies required CPU features for end-user and system VMs.
-# The features listed in this property must be separated by a blank space (e.g.: vmx vme)
+# These features must be present on the host CPU for VM deployment.
+# Multiple features should be separated by whitespace (e.g.: vmx vme).
 #guest.cpu.features=
 
 # Disables memory ballooning on VM guests for overcommit.
@@ -441,3 +442,18 @@ iscsi.session.cleanup.enabled=false
 
 # Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used
 #backoff.seconds=
+
+# Timeout (in seconds) to wait for the snapshot reversion to complete.
+# revert.snapshot.timeout=10800
+
+# Timeout (in seconds) to wait for the incremental snapshot to complete.
+# incremental.snapshot.timeout=10800
+
+# If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise.
+# create.full.clone=false
+
+# Instance conversion TMPDIR env var
+#convert.instance.env.tmpdir=
+
+# Instance conversion VIRT_V2V_TMPDIR env var
+#convert.instance.env.virtv2v.tmpdir=

agent/conf/developer.properties.template

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

agent/conf/uefi.properties.in

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Configuration file for UEFI
+
+guest.nvram.template.legacy=@GUESTNVRAMTEMPLATELEGACY@
+guest.loader.legacy=@GUESTLOADERLEGACY@
+guest.nvram.template.secure=@GUESTNVRAMTEMPLATESECURE@
+guest.loader.secure=@GUESTLOADERSECURE@
+guest.nvram.path=@GUESTNVRAMPATH@

agent/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.20.1.0</version>
+        <version>4.23.0.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

agent/src/main/java/com/cloud/agent/Agent.java

@@ -26,6 +26,8 @@ import java.net.Socket;
 import java.net.UnknownHostException;
 import java.nio.channels.ClosedChannelException;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -57,7 +59,6 @@ import org.apache.cloudstack.utils.security.KeyStoreUtils;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.ObjectUtils;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.ThreadContext;
@@ -69,6 +70,8 @@ import com.cloud.agent.api.Command;
 import com.cloud.agent.api.CronCommand;
 import com.cloud.agent.api.MaintainAnswer;
 import com.cloud.agent.api.MaintainCommand;
+import com.cloud.agent.api.MigrateAgentConnectionAnswer;
+import com.cloud.agent.api.MigrateAgentConnectionCommand;
 import com.cloud.agent.api.PingAnswer;
 import com.cloud.agent.api.PingCommand;
 import com.cloud.agent.api.ReadyCommand;
@@ -84,6 +87,7 @@ import com.cloud.resource.ResourceStatusUpdater;
 import com.cloud.resource.ServerResource;
 import com.cloud.utils.NumbersUtil;
 import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.StringUtils;
 import com.cloud.utils.concurrency.NamedThreadFactory;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.utils.exception.NioConnectionException;
@@ -93,7 +97,6 @@ import com.cloud.utils.nio.Link;
 import com.cloud.utils.nio.NioClient;
 import com.cloud.utils.nio.NioConnection;
 import com.cloud.utils.nio.Task;
-import com.cloud.utils.script.OutputInterpreter;
 import com.cloud.utils.script.Script;
 
 /**
@@ -338,9 +341,8 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                 logger.info("Attempted to connect to the server, but received an unexpected exception, trying again...", e);
             }
         }
-        shell.updateConnectedHost();
+        shell.updateConnectedHost(((NioClient)connection).getHost());
         scavengeOldAgentObjects();
-
     }
 
     public void stop(final String reason, final String detail) {
@@ -450,22 +452,30 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
         certExecutor.schedule(new PostCertificateRenewalTask(this), 5, TimeUnit.SECONDS);
     }
 
-    private void scheduleHostLBCheckerTask(final long checkInterval) {
+    private void scheduleHostLBCheckerTask(final String lbAlgorithm, final long checkInterval) {
         String name = "HostLBCheckerTask";
         if (hostLbCheckExecutor != null && !hostLbCheckExecutor.isShutdown()) {
+            logger.info("Shutting down the preferred host checker task {}", name);
             hostLbCheckExecutor.shutdown();
             try {
                 if (!hostLbCheckExecutor.awaitTermination(1, TimeUnit.SECONDS)) {
                     hostLbCheckExecutor.shutdownNow();
                 }
             } catch (InterruptedException e) {
-                logger.debug("Forcing {} shutdown as it did not shutdown in the desired time due to: {}",
+                logger.debug("Forcing the preferred host checker task {} shutdown as it did not shutdown in the desired time due to: {}",
                         name, e.getMessage());
                 hostLbCheckExecutor.shutdownNow();
             }
         }
         if (checkInterval > 0L) {
-            logger.info("Scheduling preferred host task with host.lb.interval={}ms", checkInterval);
+            if ("shuffle".equalsIgnoreCase(lbAlgorithm)) {
+                logger.info("Scheduling the preferred host checker task to trigger once (to apply lb algorithm '{}') after host.lb.interval={} ms", lbAlgorithm, checkInterval);
+                hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name)));
+                hostLbCheckExecutor.schedule(new PreferredHostCheckerTask(), checkInterval, TimeUnit.MILLISECONDS);
+                return;
+            }
+
+            logger.info("Scheduling a recurring preferred host checker task with host.lb.interval={} ms", checkInterval);
             hostLbCheckExecutor = Executors.newSingleThreadScheduledExecutor((new NamedThreadFactory(name)));
             hostLbCheckExecutor.scheduleAtFixedRate(new PreferredHostCheckerTask(), checkInterval, checkInterval,
                     TimeUnit.MILLISECONDS);
@@ -539,6 +549,10 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
     }
 
     public void sendStartup(final Link link) {
+        sendStartup(link, false);
+    }
+
+    public void sendStartup(final Link link, boolean transfer) {
         final StartupCommand[] startup = serverResource.initialize();
         if (startup != null) {
             final String msHostList = shell.getPersistentProperty(null, "host");
@@ -546,6 +560,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
             for (int i = 0; i < startup.length; i++) {
                 setupStartupCommand(startup[i]);
                 startup[i].setMSHostList(msHostList);
+                startup[i].setConnectionTransferred(transfer);
                 commands[i] = startup[i];
             }
             final Request request = new Request(id != null ? id : -1, -1, commands, false, false);
@@ -598,9 +613,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
     }
 
     protected String getAgentArch() {
-        final Script command = new Script("/usr/bin/arch", 500, logger);
+        String arch = Script.runSimpleBashScript(Script.getExecutableAbsolutePath("arch"), 2000);
-        final OutputInterpreter.OneLineParser parser = new OutputInterpreter.OneLineParser();
+        logger.debug("Arch for agent: {} found: {}", _name, arch);
-        return command.execute(parser);
+        return arch;
     }
 
     @Override
@@ -608,6 +623,46 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
         return new ServerHandler(type, link, data);
     }
 
+    protected void reconnect(final Link link) {
+        reconnect(link, null, false);
+    }
+
+    protected void reconnect(final Link link, String preferredMSHost, boolean forTransfer) {
+        if (!(forTransfer || reconnectAllowed)) {
+            logger.debug("Reconnect requested but it is not allowed {}", () -> getLinkLog(link));
+            return;
+        }
+        cancelStartupTask();
+        closeAndTerminateLink(link);
+        closeAndTerminateLink(this.link);
+        setLink(null);
+        cancelTasks();
+        serverResource.disconnected();
+        logger.info("Lost connection to host: {}. Attempting reconnection while we still have {} commands in progress.", shell.getConnectedHost(), commandsInProgress.get());
+        stopAndCleanupConnection(true);
+        String host = preferredMSHost;
+        if (org.apache.commons.lang3.StringUtils.isBlank(host)) {
+            host = shell.getNextHost();
+        }
+        List<String> avoidMSHostList = shell.getAvoidHosts();
+        do {
+            if (CollectionUtils.isEmpty(avoidMSHostList) || !avoidMSHostList.contains(host)) {
+                connection = new NioClient(getAgentName(), host, shell.getPort(), shell.getWorkers(), shell.getSslHandshakeTimeout(), this);
+                logger.info("Reconnecting to host: {}", host);
+                try {
+                    connection.start();
+                } catch (final NioConnectionException e) {
+                    logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e);
+                    stopAndCleanupConnection(false);
+                }
+            }
+            shell.getBackoffAlgorithm().waitBeforeRetry();
+            host = shell.getNextHost();
+        } while (!connection.isStartup());
+        shell.updateConnectedHost(((NioClient)connection).getHost());
+        logger.info("Connected to the host: {}", shell.getConnectedHost());
+    }
+
     protected void closeAndTerminateLink(final Link link) {
         if (link == null) {
             return;
@@ -634,35 +689,6 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
         } while (connection.isStartup());
     }
 
-    protected void reconnect(final Link link) {
-        if (!reconnectAllowed) {
-            logger.debug("Reconnect requested but it is not allowed {}", () -> getLinkLog(link));
-            return;
-        }
-        cancelStartupTask();
-        closeAndTerminateLink(link);
-        closeAndTerminateLink(this.link);
-        setLink(null);
-        cancelTasks();
-        serverResource.disconnected();
-        logger.info("Lost connection to host: {}. Attempting reconnection while we still have {} commands in progress.", shell.getConnectedHost(), commandsInProgress.get());
-        stopAndCleanupConnection(true);
-        do {
-            final String host = shell.getNextHost();
-            connection = new NioClient(getAgentName(), host, shell.getPort(), shell.getWorkers(), shell.getSslHandshakeTimeout(), this);
-            logger.info("Reconnecting to host: {}", host);
-            try {
-                connection.start();
-            } catch (final NioConnectionException e) {
-                logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e);
-                stopAndCleanupConnection(false);
-            }
-            shell.getBackoffAlgorithm().waitBeforeRetry();
-        } while (!connection.isStartup());
-        shell.updateConnectedHost();
-        logger.info("Connected to the host: {}", shell.getConnectedHost());
-    }
-
     public void processStartupAnswer(final Answer answer, final Response response, final Link link) {
         boolean answerValid = cancelStartupTask();
         final StartupAnswer startup = (StartupAnswer)answer;
@@ -709,6 +735,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                 final Command cmd = cmds[i];
                 Answer answer;
                 try {
+                    if (cmd.getContextParam("logid") != null) {
+                        ThreadContext.put("logcontextid", cmd.getContextParam("logid"));
+                    }
                     if (logger.isDebugEnabled()) {
                         if (!requestLogged) // ensures request is logged only once per method call
                         {
@@ -770,12 +799,17 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                         }
                     } else if (cmd instanceof SetupMSListCommand) {
                         answer = setupManagementServerList((SetupMSListCommand) cmd);
+                    } else if (cmd instanceof MigrateAgentConnectionCommand) {
+                        answer = migrateAgentToOtherMS((MigrateAgentConnectionCommand) cmd);
                     } else {
                         if (cmd instanceof ReadyCommand) {
                             processReadyCommand(cmd);
                         }
                         commandsInProgress.incrementAndGet();
                         try {
+                            if (cmd.isReconcile()) {
+                                cmd.setRequestSequence(request.getSequence());
+                            }
                             answer = serverResource.executeRequest(cmd);
                         } finally {
                             commandsInProgress.decrementAndGet();
@@ -901,7 +935,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
         return new SetupCertificateAnswer(true);
     }
 
-    private void processManagementServerList(final List<String> msList, final String lbAlgorithm, final Long lbCheckInterval) {
+    private void processManagementServerList(final List<String> msList, final List<String> avoidMsList, final String lbAlgorithm, final Long lbCheckInterval, final boolean triggerHostLB) {
         if (CollectionUtils.isNotEmpty(msList) && StringUtils.isNotEmpty(lbAlgorithm)) {
             try {
                 final String newMSHosts = String.format("%s%s%s", com.cloud.utils.StringUtils.toCSVList(msList), IAgentShell.hostLbAlgorithmSeparator, lbAlgorithm);
@@ -913,18 +947,73 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                 throw new CloudRuntimeException("Could not persist received management servers list", e);
             }
         }
-        if ("shuffle".equals(lbAlgorithm)) {
+        shell.setAvoidHosts(avoidMsList);
-            scheduleHostLBCheckerTask(0);
+        if (triggerHostLB) {
-        } else {
+            logger.info("Triggering the preferred host checker task now");
-            scheduleHostLBCheckerTask(shell.getLbCheckerInterval(lbCheckInterval));
+            ScheduledExecutorService hostLbExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HostLB-Executor"));
+            hostLbExecutor.schedule(new PreferredHostCheckerTask(), 0, TimeUnit.MILLISECONDS);
+            hostLbExecutor.shutdown();
         }
+        scheduleHostLBCheckerTask(lbAlgorithm, shell.getLbCheckerInterval(lbCheckInterval));
     }
 
     private Answer setupManagementServerList(final SetupMSListCommand cmd) {
-        processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
+        processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), cmd.getTriggerHostLb());
         return new SetupMSListAnswer(true);
     }
 
+    private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) {
+        try {
+            if (CollectionUtils.isNotEmpty(cmd.getMsList())) {
+                processManagementServerList(cmd.getMsList(), cmd.getAvoidMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval(), false);
+            }
+            ScheduledExecutorService migrateAgentConnectionService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("MigrateAgentConnection-Job"));
+            migrateAgentConnectionService.schedule(() -> {
+                migrateAgentConnection(cmd.getAvoidMsList());
+            }, 3, TimeUnit.SECONDS);
+            migrateAgentConnectionService.shutdown();
+        } catch (Exception e) {
+            String errMsg = "Migrate agent connection failed, due to " + e.getMessage();
+            logger.debug(errMsg, e);
+            return new MigrateAgentConnectionAnswer(errMsg);
+        }
+        return new MigrateAgentConnectionAnswer(true);
+    }
+
+    private void migrateAgentConnection(List<String> avoidMsList) {
+        final String[] msHosts = shell.getHosts();
+        if (msHosts == null || msHosts.length < 1) {
+            throw new CloudRuntimeException("Management Server hosts empty, not properly configured in agent");
+        }
+
+        List<String> msHostsList = new ArrayList<>(Arrays.asList(msHosts));
+        msHostsList.removeAll(avoidMsList);
+        if (msHostsList.isEmpty() || StringUtils.isEmpty(msHostsList.get(0))) {
+            throw new CloudRuntimeException("No other Management Server hosts to migrate");
+        }
+
+        String preferredMSHost  = null;
+        for (String msHost : msHostsList) {
+            try (final Socket socket = new Socket()) {
+                socket.connect(new InetSocketAddress(msHost, shell.getPort()), 5000);
+                preferredMSHost = msHost;
+                break;
+            } catch (final IOException e) {
+                throw new CloudRuntimeException("Management server host: " + msHost + " is not reachable, to migrate connection");
+            }
+        }
+
+        if (preferredMSHost == null) {
+            throw new CloudRuntimeException("Management server host(s) are not reachable, to migrate connection");
+        }
+
+        logger.debug("Management server host " + preferredMSHost + " is found to be reachable, trying to reconnect");
+        shell.resetHostCounter();
+        shell.setAvoidHosts(avoidMsList);
+        shell.setConnectionTransfer(true);
+        reconnect(link, preferredMSHost, true);
+    }
+
     public void processResponse(final Response response, final Link link) {
         final Answer answer = response.getAnswer();
         logger.debug("Received response: {}", response.toString());
@@ -935,14 +1024,23 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
             for (final IAgentControlListener listener : controlListeners) {
                 listener.processControlResponse(response, (AgentControlAnswer)answer);
             }
-        } else if (answer instanceof PingAnswer && (((PingAnswer) answer).isSendStartup()) && reconnectAllowed) {
+        } else if (answer instanceof PingAnswer) {
-            logger.info("Management server requested startup command to reinitialize the agent");
+            processPingAnswer((PingAnswer) answer);
-            sendStartup(link);
         } else {
             updateLastPingResponseTime();
         }
     }
 
+    private void processPingAnswer(final PingAnswer answer) {
+        if ((answer.isSendStartup()) && reconnectAllowed) {
+            logger.info("Management server requested startup command to reinitialize the agent");
+            sendStartup(link);
+        } else {
+            serverResource.processPingAnswer((PingAnswer) answer);
+        }
+        shell.setAvoidHosts(answer.getAvoidMsList());
+    }
+
     public void processReadyCommand(final Command cmd) {
         final ReadyCommand ready = (ReadyCommand)cmd;
         // Set human readable sizes;
@@ -959,7 +1057,7 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
         }
 
         verifyAgentArch(ready.getArch());
-        processManagementServerList(ready.getMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval());
+        processManagementServerList(ready.getMsHostList(), ready.getAvoidMsHostList(), ready.getLbAlgorithm(), ready.getLbCheckInterval(), false);
 
         logger.info("Ready command is processed for agent [id: {}, uuid: {}, name: {}]", getId(), getUuid(), getName());
     }
@@ -1005,6 +1103,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
             Answer answer = null;
             commandsInProgress.incrementAndGet();
             try {
+                if (command.isReconcile()) {
+                    command.setRequestSequence(req.getSequence());
+                }
                 answer = serverResource.executeRequest(command);
             } finally {
                 commandsInProgress.decrementAndGet();
@@ -1128,12 +1229,12 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
     public class WatchTask implements Runnable {
         protected Request _request;
         protected Agent _agent;
-        protected Link _link;
+        protected Link link;
 
         public WatchTask(final Link link, final Request request, final Agent agent) {
             super();
             _request = request;
-            _link = link;
+            this.link = link;
             _agent = agent;
         }
 
@@ -1142,9 +1243,9 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
             logger.trace("Scheduling {}", (_request instanceof Response ? "Ping" : "Watch Task"));
             try {
                 if (_request instanceof Response) {
-                    outRequestHandler.submit(new ServerHandler(Task.Type.OTHER, _link, _request));
+                    outRequestHandler.submit(new ServerHandler(Task.Type.OTHER, link, _request));
                 } else {
-                    _link.schedule(new ServerHandler(Task.Type.OTHER, _link, _request));
+                    link.schedule(new ServerHandler(Task.Type.OTHER, link, _request));
                 }
             } catch (final ClosedChannelException e) {
                 logger.warn("Unable to schedule task because channel is closed");
@@ -1153,12 +1254,12 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
     }
 
     public class StartupTask implements Runnable {
-        protected Link _link;
+        protected Link link;
         private final AtomicBoolean cancelled = new AtomicBoolean(false);
 
         public StartupTask(final Link link) {
             logger.debug("Startup task created");
-            _link = link;
+            this.link = link;
         }
 
         public boolean cancel() {
@@ -1176,8 +1277,8 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                 logger.info("The running startup command is now invalid. Attempting reconnect");
                 startupTask.set(null);
                 startupWait = DEFAULT_STARTUP_WAIT * 2;
-                logger.debug("Executing reconnect from task - {}", () -> getLinkLog(_link));
+                logger.debug("Executing reconnect from task - {}", () -> getLinkLog(link));
-                reconnect(_link);
+                reconnect(link);
             }
         }
     }
@@ -1210,7 +1311,8 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
             if (task.getType() == Task.Type.CONNECT) {
                 shell.getBackoffAlgorithm().reset();
                 setLink(task.getLink());
-                sendStartup(task.getLink());
+                sendStartup(task.getLink(), shell.isConnectionTransfer());
+                shell.setConnectionTransfer(false);
             } else if (task.getType() == Task.Type.DATA) {
                 Request request;
                 try {
@@ -1220,7 +1322,6 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                         processResponse((Response)request, task.getLink());
                     } else {
                         //put the requests from mgt server into another thread pool, as the request may take a longer time to finish. Don't block the NIO main thread pool
-                        //processRequest(request, task.getLink());
                         requestHandler.submit(new AgentRequestHandler(getType(), getLink(), request));
                     }
                 } catch (final ClassNotFoundException e) {
@@ -1229,7 +1330,15 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                     logger.error("Error parsing task", e);
                 }
             } else if (task.getType() == Task.Type.DISCONNECT) {
-                logger.debug("Executing disconnect task - {}", () -> getLinkLog(task.getLink()));
+                try {
+                    // an issue has been found if reconnect immediately after disconnecting.
+                    // wait 5 seconds before reconnecting
+                    logger.debug("Wait for 5 secs before reconnecting, disconnect task - {}", () -> getLinkLog(task.getLink()));
+                    Thread.sleep(5000);
+                } catch (InterruptedException e) {
+                }
+                shell.setConnectionTransfer(false);
+                logger.debug("Executing disconnect task - {} and reconnecting", () -> getLinkLog(task.getLink()));
                 reconnect(task.getLink());
             } else if (task.getType() == Task.Type.OTHER) {
                 processOtherTask(task);
@@ -1298,26 +1407,26 @@ public class Agent implements HandlerFactory, IAgentControl, AgentStatusUpdater
                 if (msList == null || msList.length < 1) {
                     return;
                 }
-                final String preferredHost  = msList[0];
+                final String preferredMSHost  = msList[0];
                 final String connectedHost = shell.getConnectedHost();
                 logger.debug("Running preferred host checker task, connected host={}, preferred host={}",
-                        connectedHost, preferredHost);
+                        connectedHost, preferredMSHost);
-                if (preferredHost == null || preferredHost.equals(connectedHost) || link == null) {
+                if (preferredMSHost == null || preferredMSHost.equals(connectedHost) || link == null) {
                     return;
                 }
                 boolean isHostUp = false;
                 try (final Socket socket = new Socket()) {
-                    socket.connect(new InetSocketAddress(preferredHost, shell.getPort()), 5000);
+                    socket.connect(new InetSocketAddress(preferredMSHost, shell.getPort()), 5000);
                     isHostUp = true;
                 } catch (final IOException e) {
-                    logger.debug("Host: {} is not reachable", preferredHost);
+                    logger.debug("Host: {} is not reachable", preferredMSHost);
                 }
                 if (isHostUp && link != null && commandsInProgress.get() == 0) {
                     if (logger.isDebugEnabled()) {
-                        logger.debug("Preferred host {} is found to be reachable, trying to reconnect", preferredHost);
+                        logger.debug("Preferred host {} is found to be reachable, trying to reconnect", preferredMSHost);
                     }
                     shell.resetHostCounter();
-                    reconnect(link);
+                    reconnect(link, preferredMSHost, false);
                 }
             } catch (Throwable t) {
                 logger.error("Error caught while attempting to connect to preferred host", t);

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -66,6 +66,7 @@ public class AgentShell implements IAgentShell, Daemon {
     private String _zone;
     private String _pod;
     private String _host;
+    private List<String> _avoidHosts;
     private String _privateIp;
     private int _port;
     private int _proxyPort;
@@ -76,9 +77,9 @@ public class AgentShell implements IAgentShell, Daemon {
     private volatile boolean _exit = false;
     private int _pingRetries;
     private final List<Agent> _agents = new ArrayList<Agent>();
-    private String hostToConnect;
     private String connectedHost;
     private Long preferredHostCheckInterval;
+    private boolean connectionTransfer = false;
     protected AgentProperties agentProperties = new AgentProperties();
 
     public AgentShell() {
@@ -120,7 +121,7 @@ public class AgentShell implements IAgentShell, Daemon {
         if (_hostCounter >= hosts.length) {
             _hostCounter = 0;
         }
-        hostToConnect = hosts[_hostCounter % hosts.length];
+        String hostToConnect = hosts[_hostCounter % hosts.length];
         _hostCounter++;
         return hostToConnect;
     }
@@ -142,11 +143,10 @@ public class AgentShell implements IAgentShell, Daemon {
     }
 
     @Override
-    public void updateConnectedHost() {
+    public void updateConnectedHost(String connectedHost) {
-        connectedHost = hostToConnect;
+        this.connectedHost = connectedHost;
     }
 
-
     @Override
     public void resetHostCounter() {
         _hostCounter = 0;
@@ -165,6 +165,16 @@ public class AgentShell implements IAgentShell, Daemon {
         }
     }
 
+    @Override
+    public void setAvoidHosts(List<String> avoidHosts) {
+        _avoidHosts = avoidHosts;
+    }
+
+    @Override
+    public List<String> getAvoidHosts() {
+        return _avoidHosts;
+    }
+
     @Override
     public String getPrivateIp() {
         return _privateIp;
@@ -217,6 +227,14 @@ public class AgentShell implements IAgentShell, Daemon {
             _storage.persist(name, value);
     }
 
+    public boolean isConnectionTransfer() {
+        return connectionTransfer;
+    }
+
+    public void setConnectionTransfer(boolean connectionTransfer) {
+        this.connectionTransfer = connectionTransfer;
+    }
+
     void loadProperties() throws ConfigurationException {
         final File file = PropertiesUtil.findConfigFile("agent.properties");
 

agent/src/main/java/com/cloud/agent/IAgentShell.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.agent;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -63,13 +64,21 @@ public interface IAgentShell {
 
     String[] getHosts();
 
+    void setAvoidHosts(List<String> hosts);
+
+    List<String> getAvoidHosts();
+
     long getLbCheckerInterval(Long receivedLbInterval);
 
-    void updateConnectedHost();
+    void updateConnectedHost(String connectedHost);
 
     String getConnectedHost();
 
     void launchNewAgent(ServerResource resource) throws ConfigurationException;
 
+    boolean isConnectionTransfer();
+
+    void setConnectionTransfer(boolean connectionTransfer);
+
     Integer getSslHandshakeTimeout();
 }

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -117,7 +117,7 @@ public class AgentProperties{
 
     /**
      * Local storage path.<br>
-     * This property allows multiple values to be entered in a single String. The differente values must be separated by commas.<br>
+     * This property allows multiple values to be entered in a single String. The different values must be separated by commas.<br>
      * Data type: String.<br>
      * Default value: <code>/var/lib/libvirt/images/</code>
      */
@@ -134,7 +134,7 @@ public class AgentProperties{
 
     /**
      * MANDATORY: The UUID for the local storage pool.<br>
-     * This property allows multiple values to be entered in a single String. The differente values must be separated by commas.<br>
+     * This property allows multiple values to be entered in a single String. The different values must be separated by commas.<br>
      * Data type: String.<br>
      * Default value: <code>null</code>
      */
@@ -155,6 +155,14 @@ public class AgentProperties{
      */
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
+    /**
+     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * This configuration is only considered if libvirt.events.enabled is also true. <br>
+     * Data type: Integer.<br>
+     * Default value: <code>259200</code>
+     */
+    public static final Property<Integer> QCOW2_DELTA_MERGE_TIMEOUT = new Property<>("qcow2.delta.merge.timeout", 60 * 60 * 72);
+
     /**
      * This parameter sets the VM migration speed (in mbps). The default value is -1,<br>
      * which means that the agent will try to guess the speed of the guest network and consume all possible bandwidth.<br>
@@ -213,6 +221,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.script", "libvirt-vm-xml-transformer.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM XML transformer shell script.<br>
+     * The shell script is used to execute the Libvirt VM XML transformer script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-xml-transformer.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.shell_script", "libvirt-vm-xml-transformer.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_xml_transformer.script properties to define the Libvirt VM XML transformer method.<br>
      * Libvirt XML transformer hook does XML-to-XML transformation.<br>
@@ -233,6 +250,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on start shell script.<br>
+     * The shell script is used to execute the Libvirt VM on start script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_start.script properties to define the Libvirt VM on start method.<br>
      * The hook is called right after Libvirt successfully launched the VM.<br>
@@ -252,6 +278,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on stop shell script.<br>
+     * The shell script is used to execute the Libvirt VM on stop script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_stop.script properties to define the Libvirt VM on stop method.<br>
      * The hook is called right after libvirt successfully stopped the VM.<br>
@@ -383,15 +418,16 @@ public class AgentProperties{
     /**
      * This param will set the CPU architecture for the domain to override what the management server would send.<br>
      * In case of arm64 (aarch64), this will change the machine type to 'virt' and add a SCSI and a USB controller in the domain XML.<br>
-     * Possible values: x86_64 | aarch64 <br>
+     * Possible values: x86_64 | aarch64 | s390x <br>
      * Data type: String.<br>
      * Default value: <code>null</code> (will set use the architecture of the VM's OS).
      */
     public static final Property<String> GUEST_CPU_ARCH = new Property<>("guest.cpu.arch", null, String.class);
 
     /**
-     * This param will require CPU features on the CPU section.<br>
+     * Specifies required CPU features for end-user and system VMs.<br>
-     * The features listed in this property must be separated by a blank space (see example below).<br>
+     * These features must be present on the host CPU for VM deployment.<br>
+     * Multiple features should be separated by whitespace (see example below).<br>
      * Possible values: vmx vme <br>
      * Data type: String.<br>
      * Default value: <code>null</code>
@@ -758,6 +794,20 @@ public class AgentProperties{
      */
     public static final Property<Boolean> VIRTV2V_VERBOSE_ENABLED = new Property<>("virtv2v.verbose.enabled", false);
 
+    /**
+     * Set env TMPDIR var for virt-v2v Instance Conversion from VMware to KVM
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> CONVERT_ENV_TMPDIR = new Property<>("convert.instance.env.tmpdir", null, String.class);
+
+    /**
+     * Set env VIRT_V2V_TMPDIR var for virt-v2v Instance Conversion from VMware to KVM
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> CONVERT_ENV_VIRTV2V_TMPDIR = new Property<>("convert.instance.env.virtv2v.tmpdir", null, String.class);
+
     /**
      * BGP controll CIDR
      * Data type: String.<br>
@@ -816,14 +866,32 @@ public class AgentProperties{
      * Data type: Integer.<br>
      * Default value: <code>null</code>
      */
-    public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", null, Integer.class);
+    public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", 30, Integer.class);
+
+    /**
+     * Timeout (in seconds) to wait for the incremental snapshot to complete.
+     * */
+    public static final Property<Integer> INCREMENTAL_SNAPSHOT_TIMEOUT = new Property<>("incremental.snapshot.timeout", 10800);
+
+    /**
+     * Timeout (in seconds) to wait for the snapshot reversion to complete.
+     * */
+    public static final Property<Integer> REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800);
+
+    /**
+     *  If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise. <br>
+     * Data type: Boolean. <br>
+     * Default value: <code>false</code>
+     */
+    public static final Property<Boolean> CREATE_FULL_CLONE = new Property<>("create.full.clone", false);
+
 
     public static class Property <T>{
         private String name;
         private T defaultValue;
         private Class<T> typeClass;
 
-        Property(String name, T value) {
+        public Property(String name, T value) {
             init(name, value);
         }
 

agent/src/main/java/com/cloud/agent/resource/DummyResource.java

@@ -20,7 +20,6 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 
 import com.cloud.agent.IAgentControl;
 import com.cloud.agent.api.Answer;
@@ -40,6 +39,7 @@ import com.cloud.resource.ServerResource;
 import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.utils.StringUtils;
+import com.cloud.utils.UuidUtils;
 
 public class DummyResource implements ServerResource {
     String _name;
@@ -133,7 +133,7 @@ public class DummyResource implements ServerResource {
         String hostIp = getConfiguredProperty("private.ip.address", "127.0.0.1");
         String localStoragePath = getConfiguredProperty("local.storage.path", "/mnt");
         String lh = hostIp + localStoragePath;
-        String uuid = UUID.nameUUIDFromBytes(lh.getBytes(StringUtils.getPreferredCharset())).toString();
+        String uuid = UuidUtils.nameUUIDFromBytes(lh.getBytes(StringUtils.getPreferredCharset())).toString();
 
         String capacity = getConfiguredProperty("local.storage.capacity", "1000000000");
         String available = getConfiguredProperty("local.storage.avail", "10000000");

agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java

@@ -175,12 +175,12 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe
                 try {
                     is.close();
                 } catch (final IOException e) {
-                    logger.warn("Exception when closing , console proxy address : {}", proxyManagementIp);
+                    logger.warn("Exception when closing , console proxy address: {}", proxyManagementIp);
                     success = false;
                 }
             }
         } catch (final IOException e) {
-            logger.warn("Unable to open console proxy command port url, console proxy address : {}", proxyManagementIp);
+            logger.warn("Unable to open console proxy command port url, console proxy address: {}", proxyManagementIp);
             success = false;
         }
 
@@ -397,9 +397,8 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe
     }
 
     public String authenticateConsoleAccess(String host, String port, String vmId, String sid, String ticket,
-                                            Boolean isReauthentication, String sessionToken) {
+                                            Boolean isReauthentication, String sessionToken, String clientAddress) {
-
+        ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken, clientAddress);
-        ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken);
         cmd.setReauthenticating(isReauthentication);
 
         ConsoleProxyAuthenticationResult result = new ConsoleProxyAuthenticationResult();

agent/src/test/java/com/cloud/agent/AgentShellTest.java

@@ -358,7 +358,7 @@ public class AgentShellTest {
         AgentShell shell = new AgentShell();
         shell.setHosts("test");
         shell.getNextHost();
-        shell.updateConnectedHost();
+        shell.updateConnectedHost("test");
 
         Assert.assertEquals(expected, shell.getConnectedHost());
     }

api/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.20.1.0</version>
+        <version>4.23.0.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

api/src/main/java/com/cloud/agent/api/Command.java

@@ -19,9 +19,10 @@ package com.cloud.agent.api;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.cloud.agent.api.LogLevel.Log4jLevel;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
 
 /**
  * implemented by classes that extends the Command class. Command specifies
@@ -35,6 +36,23 @@ public abstract class Command {
         Continue, Stop
     }
 
+    public enum State {
+        CREATED,        // Command is created by management server
+        STARTED,        // Command is started by agent
+        PROCESSING,     // Processing by agent
+        PROCESSING_IN_BACKEND,  // Processing in backend by agent
+        COMPLETED,      // Operation succeeds by agent or management server
+        FAILED,         // Operation fails by agent
+        RECONCILE_RETRY,        // Ready for retry of reconciliation
+        RECONCILING,    // Being reconciled by management server
+        RECONCILED,     // Reconciled by management server
+        RECONCILE_SKIPPED,  // Skip the reconciliation as the resource state is inconsistent with the command
+        RECONCILE_FAILED,       // Fail to reconcile by management server
+        TIMED_OUT,      // Timed out on management server or agent
+        INTERRUPTED,    // Interrupted by management server or agent (for example agent is restarted),
+        DANGLED_IN_BACKEND     // Backend process which cannot be processed normally (for example agent is restarted)
+    }
+
     public static final String HYPERVISOR_TYPE = "hypervisorType";
 
     // allow command to carry over hypervisor or other environment related context info
@@ -42,6 +60,8 @@ public abstract class Command {
     protected Map<String, String> contextMap = new HashMap<String, String>();
     private int wait;  //in second
     private boolean bypassHostMaintenance = false;
+    private transient long requestSequence = 0L;
+    protected Map<String, Map<String, String>> externalDetails;
 
     protected Command() {
         this.wait = 0;
@@ -82,6 +102,10 @@ public abstract class Command {
         return contextMap.get(name);
     }
 
+    public Map<String, String> getContextMap() {
+        return contextMap;
+    }
+
     public boolean allowCaching() {
         return true;
     }
@@ -94,6 +118,26 @@ public abstract class Command {
         this.bypassHostMaintenance = bypassHostMaintenance;
     }
 
+    public boolean isReconcile() {
+        return false;
+    }
+
+    public long getRequestSequence() {
+        return requestSequence;
+    }
+
+    public void setRequestSequence(long requestSequence) {
+        this.requestSequence = requestSequence;
+    }
+
+    public void setExternalDetails(Map<String, Map<String, String>> externalDetails) {
+        this.externalDetails = externalDetails;
+    }
+
+    public Map<String, Map<String, String>> getExternalDetails() {
+        return externalDetails;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

api/src/main/java/com/cloud/agent/api/VgpuTypesInfo.java

@@ -15,10 +15,24 @@
 // specific language governing permissions and limitations
 // under the License.
 package com.cloud.agent.api;
+
+import org.apache.cloudstack.gpu.GpuDevice;
+
 public class VgpuTypesInfo {
 
+    private boolean passthroughEnabled = true;
+    private GpuDevice.DeviceType deviceType;
+    private String parentBusAddress;
+    private String busAddress;
+    private String numaNode;
+    private String pciRoot;
+    private String deviceId;
+    private String deviceName;
+    private String vendorId;
+    private String vendorName;
     private String modelName;
     private String groupName;
+    private String vmName;
     private Long maxHeads;
     private Long videoRam;
     private Long maxResolutionX;
@@ -26,6 +40,7 @@ public class VgpuTypesInfo {
     private Long maxVgpuPerGpu;
     private Long remainingCapacity;
     private Long maxCapacity;
+    private boolean display = false;
 
     public String getModelName() {
         return modelName;
@@ -39,22 +54,42 @@ public class VgpuTypesInfo {
         return videoRam;
     }
 
+    public void setVideoRam(Long videoRam) {
+        this.videoRam = videoRam;
+    }
+
     public Long getMaxHeads() {
         return maxHeads;
     }
 
+    public void setMaxHeads(Long maxHeads) {
+        this.maxHeads = maxHeads;
+    }
+
     public Long getMaxResolutionX() {
         return maxResolutionX;
     }
 
+    public void setMaxResolutionX(Long maxResolutionX) {
+        this.maxResolutionX = maxResolutionX;
+    }
+
     public Long getMaxResolutionY() {
         return maxResolutionY;
     }
 
+    public void setMaxResolutionY(Long maxResolutionY) {
+        this.maxResolutionY = maxResolutionY;
+    }
+
     public Long getMaxVpuPerGpu() {
         return maxVgpuPerGpu;
     }
 
+    public void setMaxVgpuPerGpu(Long maxVgpuPerGpu) {
+        this.maxVgpuPerGpu = maxVgpuPerGpu;
+    }
+
     public Long getRemainingCapacity() {
         return remainingCapacity;
     }
@@ -71,8 +106,133 @@ public class VgpuTypesInfo {
         this.maxCapacity = maxCapacity;
     }
 
-    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX, Long maxResolutionY, Long maxVgpuPerGpu,
+    public boolean isPassthroughEnabled() {
-            Long remainingCapacity, Long maxCapacity) {
+        return passthroughEnabled;
+    }
+
+    public void setPassthroughEnabled(boolean passthroughEnabled) {
+        this.passthroughEnabled = passthroughEnabled;
+    }
+
+    public GpuDevice.DeviceType getDeviceType() {
+        return deviceType;
+    }
+
+    public void setDeviceType(GpuDevice.DeviceType deviceType) {
+        this.deviceType = deviceType;
+    }
+
+    public String getParentBusAddress() {
+        return parentBusAddress;
+    }
+
+    public void setParentBusAddress(String parentBusAddress) {
+        this.parentBusAddress = parentBusAddress;
+    }
+
+    public String getBusAddress() {
+        return busAddress;
+    }
+
+    public void setBusAddress(String busAddress) {
+        this.busAddress = busAddress;
+    }
+
+    public String getNumaNode() {
+        return numaNode;
+    }
+
+    public void setNumaNode(String numaNode) {
+        this.numaNode = numaNode;
+    }
+
+    public String getPciRoot() {
+        return pciRoot;
+    }
+
+    public void setPciRoot(String pciRoot) {
+        this.pciRoot = pciRoot;
+    }
+
+    public String getDeviceId() {
+        return deviceId;
+    }
+
+    public void setDeviceId(String deviceId) {
+        this.deviceId = deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public void setDeviceName(String deviceName) {
+        this.deviceName = deviceName;
+    }
+
+    public String getVendorId() {
+        return vendorId;
+    }
+
+    public void setVendorId(String vendorId) {
+        this.vendorId = vendorId;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    public void setVendorName(String vendorName) {
+        this.vendorName = vendorName;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+
+    public void setVmName(String vmName) {
+        this.vmName = vmName;
+    }
+
+    public boolean isDisplay() {
+        return display;
+    }
+
+    public void setDisplay(boolean display) {
+        this.display = display;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+            String vendorId, String vendorName, String deviceId, String deviceName, String numaNode, String pciRoot
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+        this.numaNode = numaNode;
+        this.pciRoot = pciRoot;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+                         String vendorId, String vendorName, String deviceId, String deviceName
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+    }
+
+    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX,
+            Long maxResolutionY, Long maxVgpuPerGpu, Long remainingCapacity, Long maxCapacity
+    ) {
         this.groupName = groupName;
         this.modelName = modelName;
         this.videoRam = videoRam;

api/src/main/java/com/cloud/agent/api/storage/OVFHelper.java

@@ -119,8 +119,7 @@ public class OVFHelper {
         boolean password = StringUtils.isNotBlank(passStr) && passStr.equalsIgnoreCase("true");
         String label = ovfParser.getChildNodeValue(node, "Label");
         String description = ovfParser.getChildNodeValue(node, "Description");
-        logger.debug("Creating OVF property index " + index + (category == null ? "" : " for category " + category)
+        logger.debug("Creating OVF property index {} {} with key = {}", index, (category == null ? "" : " for category " + category), key);
-                + " with key = " + key);
         return new OVFPropertyTO(key, type, value, qualifiers, userConfigurable,
                 label, description, password, index, category);
     }
@@ -152,7 +151,7 @@ public class OVFHelper {
                     if (child.getNodeName().equalsIgnoreCase("Category") ||
                             child.getNodeName().endsWith(":Category")) {
                         lastCategoryFound = child.getTextContent();
-                        logger.info("Category found " + lastCategoryFound);
+                        logger.info("Category found {}", lastCategoryFound);
                     } else if (child.getNodeName().equalsIgnoreCase("Property") ||
                             child.getNodeName().endsWith(":Property")) {
                         OVFPropertyTO prop = createOVFPropertyFromNode(child, propertyIndex, lastCategoryFound);
@@ -250,13 +249,13 @@ public class OVFHelper {
         int diskNumber = 0;
         for (OVFVirtualHardwareItemTO diskItem : diskHardwareItems) {
             if (StringUtils.isBlank(diskItem.getHostResource())) {
-                logger.error("Missing disk information for hardware item " + diskItem.getElementName() + " " + diskItem.getInstanceId());
+                logger.error("Missing disk information for hardware item {} {}", diskItem.getElementName(), diskItem.getInstanceId());
                 continue;
             }
             String diskId = extractDiskIdFromDiskHostResource(diskItem.getHostResource());
             OVFDisk diskDefinition = getDiskDefinitionFromDiskId(diskId, disks);
             if (diskDefinition == null) {
-                logger.error("Missing disk definition for disk ID " + diskId);
+                logger.error("Missing disk definition for disk ID {}", diskId);
             }
             OVFFile fileDefinition = getFileDefinitionFromDiskDefinition(diskDefinition._fileRef, files);
             DatadiskTO datadiskTO = generateDiskTO(fileDefinition, diskDefinition, ovfParentPath, diskNumber, diskItem);
@@ -278,8 +277,8 @@ public class OVFHelper {
         if (StringUtils.isNotBlank(path)) {
             File f = new File(path);
             if (!f.exists() || f.isDirectory()) {
-                logger.error("One of the attached disk or iso does not exists " + path);
+                logger.error("One of the attached disk or ISOs does not exists {}", path);
-                throw new InternalErrorException("One of the attached disk or iso as stated on OVF does not exists " + path);
+                throw new InternalErrorException("One of the attached disk or ISOs as stated on OVF does not exists " + path);
             }
         }
         Long capacity = disk != null ? disk._capacity : file._size;
@@ -334,9 +333,7 @@ public class OVFHelper {
             od._controller = getControllerType(items, od._diskId);
             vd.add(od);
         }
-        if (logger.isTraceEnabled()) {
+        logger.trace("Found {} disk definitions", vd.size());
-            logger.trace(String.format("found %d disk definitions",vd.size()));
-        }
         return vd;
     }
 
@@ -366,9 +363,7 @@ public class OVFHelper {
                 vf.add(of);
             }
         }
-        if (logger.isTraceEnabled()) {
+        logger.trace("Found {} file definitions in {}", vf.size(), ovfFile.getPath());
-            logger.trace(String.format("found %d file definitions in %s",vf.size(), ovfFile.getPath()));
-        }
         return vf;
     }
 
@@ -506,7 +501,7 @@ public class OVFHelper {
             outfile.write(writer.toString());
             outfile.close();
         } catch (IOException | TransformerException e) {
-            logger.info("Unexpected exception caught while rewriting OVF:" + e.getMessage(), e);
+            logger.info("Unexpected exception caught while rewriting OVF: {}", e.getMessage(), e);
             throw new CloudRuntimeException(e);
         }
     }
@@ -522,9 +517,7 @@ public class OVFHelper {
 
     public List<OVFNetworkTO> getNetPrerequisitesFromDocument(Document doc) throws InternalErrorException {
         if (doc == null) {
-            if (logger.isTraceEnabled()) {
+            logger.trace("No document to parse; returning no prerequisite networks");
-                logger.trace("no document to parse; returning no prerequisite networks");
-            }
             return Collections.emptyList();
         }
 
@@ -540,9 +533,7 @@ public class OVFHelper {
     private void matchNicsToNets(Map<String, OVFNetworkTO> nets, Node systemElement) {
         final DocumentTraversal traversal = (DocumentTraversal) systemElement;
         final NodeIterator iterator = traversal.createNodeIterator(systemElement, NodeFilter.SHOW_ELEMENT, null, true);
-        if (logger.isTraceEnabled()) {
+        logger.trace("Starting out with {} network-prerequisites, parsing hardware", nets.size());
-            logger.trace(String.format("starting out with %d network-prerequisites, parsing hardware",nets.size()));
-        }
         int nicCount = 0;
         for (Node n = iterator.nextNode(); n != null; n = iterator.nextNode()) {
             final Element e = (Element) n;
@@ -550,9 +541,7 @@ public class OVFHelper {
                 nicCount++;
                 String name = e.getTextContent(); // should be in our nets
                 if(nets.get(name) == null) {
-                    if(logger.isInfoEnabled()) {
+                    logger.info("Found a NIC definition without a Network definition by name {}, adding it to the list.", name);
-                        logger.info(String.format("found a nic definition without a network definition byname %s, adding it to the list.", name));
-                    }
                     nets.put(name, new OVFNetworkTO());
                 }
                 OVFNetworkTO thisNet = nets.get(name);
@@ -561,9 +550,7 @@ public class OVFHelper {
                 }
             }
         }
-        if (logger.isTraceEnabled()) {
+        logger.trace("Ending up with {} network-prerequisites, parsed {} nics", nets.size(), nicCount);
-            logger.trace(String.format("ending up with %d network-prerequisites, parsed %d nics", nets.size(), nicCount));
-        }
     }
 
     /**
@@ -585,7 +572,7 @@ public class OVFHelper {
             int addressOnParent = Integer.parseInt(addressOnParentStr);
             nic.setAddressOnParent(addressOnParent);
         } catch (NumberFormatException e) {
-            logger.warn("Encountered element of type \"AddressOnParent\", that could not be parse to an integer number: " + addressOnParentStr);
+            logger.warn("Encountered element of type \"AddressOnParent\", that could not be parse to an integer number: {}", addressOnParentStr);
         }
 
         boolean automaticAllocation = StringUtils.isNotBlank(automaticAllocationStr) && Boolean.parseBoolean(automaticAllocationStr);
@@ -597,7 +584,7 @@ public class OVFHelper {
             int instanceId = Integer.parseInt(instanceIdStr);
             nic.setInstanceID(instanceId);
         } catch (NumberFormatException e) {
-            logger.warn("Encountered element of type \"InstanceID\", that could not be parse to an integer number: " + instanceIdStr);
+            logger.warn("Encountered element of type \"InstanceID\", that could not be parse to an integer number: {}", instanceIdStr);
         }
 
         nic.setResourceSubType(resourceSubType);
@@ -630,9 +617,7 @@ public class OVFHelper {
 
             nets.put(networkName,network);
         }
-        if (logger.isTraceEnabled()) {
+        logger.trace("Found {} Networks in Template", nets.size());
-            logger.trace(String.format("found %d networks in template", nets.size()));
-        }
         return nets;
     }
 
@@ -771,7 +756,7 @@ public class OVFHelper {
             try {
                 return Long.parseLong(value);
             } catch (NumberFormatException e) {
-                logger.debug("Could not parse the value: " + value + ", ignoring it");
+                logger.debug("Could not parse the value: {}, ignoring it", value);
             }
         }
         return null;
@@ -782,7 +767,7 @@ public class OVFHelper {
             try {
                 return Integer.parseInt(value);
             } catch (NumberFormatException e) {
-                logger.debug("Could not parse the value: " + value + ", ignoring it");
+                logger.debug("Could not parse the value: {}, ignoring it", value);
             }
         }
         return null;
@@ -820,7 +805,7 @@ public class OVFHelper {
                 try {
                     compressedLicense = compressOVFEula(eulaLicense);
                 } catch (IOException e) {
-                    logger.error("Could not compress the license for info " + eulaInfo);
+                    logger.error("Could not compress the license for info {}", eulaInfo);
                     continue;
                 }
                 OVFEulaSectionTO eula = new OVFEulaSectionTO(eulaInfo, compressedLicense, eulaIndex);

api/src/main/java/com/cloud/agent/api/storage/OVFParser.java

@@ -54,7 +54,7 @@ public class OVFParser {
             documentBuilderFactory.setNamespaceAware(true);
             documentBuilder = documentBuilderFactory.newDocumentBuilder();
         } catch (ParserConfigurationException e) {
-            logger.error("Cannot start the OVF parser: " + e.getMessage(), e);
+            logger.error("Cannot start the OVF parser: {}", e.getMessage(), e);
         }
     }
 
@@ -70,7 +70,7 @@ public class OVFParser {
         try {
             return documentBuilder.parse(new File(ovfFilePath));
         } catch (SAXException | IOException e) {
-            logger.error("Error parsing " + ovfFilePath + " " + e.getMessage(), e);
+            logger.error("Error parsing {} {}", ovfFilePath, e.getMessage(), e);
             return null;
         }
     }

api/src/main/java/com/cloud/agent/api/to/DiskTO.java

@@ -46,7 +46,7 @@ public class DiskTO {
     private Long diskSeq;
     private String path;
     private Volume.Type type;
-    private Map<String, String> _details;
+    private Map<String, String> details;
 
     public DiskTO() {
 
@@ -92,10 +92,10 @@ public class DiskTO {
     }
 
     public void setDetails(Map<String, String> details) {
-        _details = details;
+        this.details = details;
     }
 
     public Map<String, String> getDetails() {
-        return _details;
+        return details;
     }
 }

api/src/main/java/com/cloud/agent/api/to/FirewallRuleTO.java

@@ -47,7 +47,7 @@ public class FirewallRuleTO implements InternalIdentity {
     int[] srcPortRange;
     boolean revoked;
     boolean alreadyAdded;
-    private List<String> sourceCidrList;
+    protected List<String> sourceCidrList;
     private List<String> destCidrList;
     FirewallRule.Purpose purpose;
     private Integer icmpType;

api/src/main/java/com/cloud/agent/api/to/GPUDeviceTO.java

@@ -16,7 +16,9 @@
 // under the License.
 package com.cloud.agent.api.to;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 import com.cloud.agent.api.VgpuTypesInfo;
 
@@ -24,9 +26,23 @@ public class GPUDeviceTO {
 
     private String gpuGroup;
     private String vgpuType;
+    private int gpuCount;
     private HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = new HashMap<String, HashMap<String, VgpuTypesInfo>>();
+    private List<VgpuTypesInfo> gpuDevices = new ArrayList<>();
 
-    public GPUDeviceTO( String gpuGroup, String vgpuType, HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
+    public GPUDeviceTO(String gpuGroup, String vgpuType, int gpuCount,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails,
+                       List<VgpuTypesInfo> gpuDevices) {
+        this.gpuGroup = gpuGroup;
+        this.vgpuType = vgpuType;
+        this.groupDetails = groupDetails;
+        this.gpuCount = gpuCount;
+        this.gpuDevices = gpuDevices;
+
+    }
+
+    public GPUDeviceTO(String gpuGroup, String vgpuType,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
         this.gpuGroup = gpuGroup;
         this.vgpuType = vgpuType;
         this.groupDetails = groupDetails;
@@ -48,6 +64,14 @@ public class GPUDeviceTO {
         this.vgpuType = vgpuType;
     }
 
+    public int getGpuCount() {
+        return gpuCount;
+    }
+
+    public void setGpuCount(int gpuCount) {
+        this.gpuCount = gpuCount;
+    }
+
     public HashMap<String, HashMap<String, VgpuTypesInfo>> getGroupDetails() {
         return groupDetails;
     }
@@ -56,4 +80,11 @@ public class GPUDeviceTO {
         this.groupDetails = groupDetails;
     }
 
+    public List<VgpuTypesInfo> getGpuDevices() {
+        return gpuDevices;
+    }
+
+    public void setGpuDevices(List<VgpuTypesInfo> gpuDevices) {
+        this.gpuDevices = gpuDevices;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java

@@ -71,7 +71,7 @@ public class LoadBalancerTO {
         this.destinations = new DestinationTO[destinations.size()];
         this.stickinessPolicies = null;
         this.sslCert = null;
-        this.lbProtocol = null;
+        this.lbProtocol = protocol;
         int i = 0;
         for (LbDestination destination : destinations) {
             this.destinations[i++] = new DestinationTO(destination.getIpAddress(), destination.getDestinationPortStart(), destination.isRevoked(), false);
@@ -205,6 +205,10 @@ public class LoadBalancerTO {
         return this.sslCert;
     }
 
+    public void setLbSslCert(LbSslCert sslCert) {
+        this.sslCert = sslCert;
+    }
+
     public String getSrcIpVlan() {
         return srcIpVlan;
     }

api/src/main/java/com/cloud/agent/api/to/NetworkTO.java

@@ -36,7 +36,7 @@ public class NetworkTO {
     protected TrafficType type;
     protected URI broadcastUri;
     protected URI isolationUri;
-    protected boolean isSecurityGroupEnabled;
+    protected boolean securityGroupEnabled;
     protected String name;
     protected String ip6address;
     protected String ip6gateway;
@@ -112,7 +112,7 @@ public class NetworkTO {
     }
 
     public void setSecurityGroupEnabled(boolean enabled) {
-        this.isSecurityGroupEnabled = enabled;
+        this.securityGroupEnabled = enabled;
     }
 
     /**
@@ -221,7 +221,7 @@ public class NetworkTO {
     }
 
     public boolean isSecurityGroupEnabled() {
-        return this.isSecurityGroupEnabled;
+        return this.securityGroupEnabled;
     }
 
     public void setIp6Dns1(String ip6Dns1) {

api/src/main/java/com/cloud/agent/api/to/NicTO.java

@@ -86,6 +86,14 @@ public class NicTO extends NetworkTO {
         this.nicUuid = uuid;
     }
 
+    public String getNicUuid() {
+        return nicUuid;
+    }
+
+    public void setNicUuid(String nicUuid) {
+        this.nicUuid = nicUuid;
+    }
+
     @Override
     public String toString() {
         return new StringBuilder("[Nic:").append(type).append("-").append(ip).append("-").append(broadcastUri).append("]").toString();

api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java

@@ -21,8 +21,6 @@ import com.cloud.network.rules.PortForwardingRule;
 import com.cloud.utils.net.NetUtils;
 import org.apache.commons.lang3.StringUtils;
 
-import java.util.List;
-
 /**
  * PortForwardingRuleTO specifies one port forwarding rule.
  *
@@ -32,8 +30,6 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
     String dstIp;
     int[] dstPortRange;
 
-    List<String> sourceCidrList;
-
     protected PortForwardingRuleTO() {
         super();
     }

api/src/main/java/com/cloud/agent/api/to/VirtualMachineMetadataTO.java

@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class VirtualMachineMetadataTO {
+    // VM details
+    private final String name;
+    private final String internalName;
+    private final String displayName;
+    private final String instanceUuid;
+    private final Integer cpuCores;
+    private final Integer memory;
+    private final Long created;
+    private final Long started;
+
+    // Owner details
+    private final String ownerDomainUuid;
+    private final String ownerDomainName;
+    private final String ownerAccountUuid;
+    private final String ownerAccountName;
+    private final String ownerProjectUuid;
+    private final String ownerProjectName;
+
+    // Host and service offering
+    private final String serviceOfferingName;
+    private final List<String> serviceOfferingHostTags;
+
+    // zone, pod, and cluster details
+    private final String zoneName;
+    private final String zoneUuid;
+    private final String podName;
+    private final String podUuid;
+    private final String clusterName;
+    private final String clusterUuid;
+
+    // resource tags
+    private final Map<String, String> resourceTags;
+
+    public VirtualMachineMetadataTO(
+            String name, String internalName, String displayName, String instanceUuid, Integer cpuCores, Integer memory, Long created, Long started,
+            String ownerDomainUuid, String ownerDomainName, String ownerAccountUuid, String ownerAccountName, String ownerProjectUuid, String ownerProjectName,
+            String serviceOfferingName, List<String> serviceOfferingHostTags,
+            String zoneName, String zoneUuid, String podName, String podUuid, String clusterName, String clusterUuid, Map<String, String> resourceTags) {
+        /*
+        * Something failed in the metadata shall not be a fatal error, the VM can still be started
+        * Thus, the unknown fields just get an explicit "unknown" value so it can be fixed in case
+        * there are bugs on some execution paths.
+        * */
+
+        this.name = (name != null) ? name : "unknown";
+        this.internalName = (internalName != null) ? internalName : "unknown";
+        this.displayName = (displayName != null) ? displayName : "unknown";
+        this.instanceUuid = (instanceUuid != null) ? instanceUuid : "unknown";
+        this.cpuCores = (cpuCores != null) ? cpuCores : -1;
+        this.memory = (memory != null) ? memory : -1;
+        this.created = (created != null) ? created : 0;
+        this.started = (started != null) ? started : 0;
+        this.ownerDomainUuid = (ownerDomainUuid != null) ? ownerDomainUuid : "unknown";
+        this.ownerDomainName = (ownerDomainName != null) ? ownerDomainName : "unknown";
+        this.ownerAccountUuid = (ownerAccountUuid != null) ? ownerAccountUuid : "unknown";
+        this.ownerAccountName = (ownerAccountName != null) ? ownerAccountName : "unknown";
+        this.ownerProjectUuid = (ownerProjectUuid != null) ? ownerProjectUuid : "unknown";
+        this.ownerProjectName = (ownerProjectName != null) ? ownerProjectName : "unknown";
+        this.serviceOfferingName = (serviceOfferingName != null) ? serviceOfferingName : "unknown";
+        this.serviceOfferingHostTags = (serviceOfferingHostTags != null) ? serviceOfferingHostTags : new ArrayList<>();
+        this.zoneName = (zoneName != null) ? zoneName : "unknown";
+        this.zoneUuid = (zoneUuid != null) ? zoneUuid : "unknown";
+        this.podName = (podName != null) ? podName : "unknown";
+        this.podUuid = (podUuid != null) ? podUuid : "unknown";
+        this.clusterName = (clusterName != null) ? clusterName : "unknown";
+        this.clusterUuid = (clusterUuid != null) ? clusterUuid : "unknown";
+
+        this.resourceTags = (resourceTags != null) ? resourceTags : new HashMap<>();
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getInternalName() {
+        return internalName;
+    }
+
+    public String getDisplayName() {
+        return displayName;
+    }
+
+    public String getInstanceUuid() {
+        return instanceUuid;
+    }
+
+    public Integer getCpuCores() {
+        return cpuCores;
+    }
+
+    public Integer getMemory() {
+        return memory;
+    }
+
+    public Long getCreated() { return created; }
+
+    public Long getStarted() {
+        return started;
+    }
+
+    public String getOwnerDomainUuid() {
+        return ownerDomainUuid;
+    }
+
+    public String getOwnerDomainName() {
+        return ownerDomainName;
+    }
+
+    public String getOwnerAccountUuid() {
+        return ownerAccountUuid;
+    }
+
+    public String getOwnerAccountName() {
+        return ownerAccountName;
+    }
+
+    public String getOwnerProjectUuid() {
+        return ownerProjectUuid;
+    }
+
+    public String getOwnerProjectName() {
+        return ownerProjectName;
+    }
+
+    public String getserviceOfferingName() {
+        return serviceOfferingName;
+    }
+
+    public List<String> getserviceOfferingHostTags() {
+        return serviceOfferingHostTags;
+    }
+
+    public String getZoneName() {
+        return zoneName;
+    }
+
+    public String getZoneUuid() {
+        return zoneUuid;
+    }
+
+    public String getPodName() {
+        return podName;
+    }
+
+    public String getPodUuid() {
+        return podUuid;
+    }
+
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public String getClusterUuid() {
+        return clusterUuid;
+    }
+
+    public Map<String, String> getResourceTags() { return resourceTags; }
+}

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -19,20 +19,22 @@ package com.cloud.agent.api.to;
 import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.stream.Collectors;
 
 import com.cloud.agent.api.LogLevel;
 import com.cloud.network.element.NetworkElement;
 import com.cloud.template.VirtualMachineTemplate.BootloaderType;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Type;
+import com.cloud.vm.VmDetailConstants;
 
 public class VirtualMachineTO {
     private long id;
     private String name;
     private BootloaderType bootloader;
     private VirtualMachine.State state;
-    Type type;
+    private Type type;
-    int cpus;
+    private int cpus;
 
     /**
         'speed' is still here since 4.0.X/4.1.X management servers do not support
@@ -43,49 +45,51 @@ public class VirtualMachineTO {
          So this is here for backwards compatibility with 4.0.X/4.1.X management servers
          and newer agents.
     */
-    Integer speed;
+    private Integer speed;
-    Integer minSpeed;
+    private Integer minSpeed;
-    Integer maxSpeed;
+    private Integer maxSpeed;
 
-    long minRam;
+    private long minRam;
-    long maxRam;
+    private long maxRam;
-    String hostName;
+    private String hostName;
-    String arch;
+    private String arch;
-    String os;
+    private String os;
-    String platformEmulator;
+    private String platformEmulator;
-    String bootArgs;
+    private String bootArgs;
-    String[] bootupScripts;
+    private String[] bootupScripts;
-    boolean enableHA;
+    private boolean enableHA;
-    boolean limitCpuUse;
+    private boolean limitCpuUse;
-    boolean enableDynamicallyScaleVm;
+    private boolean enableDynamicallyScaleVm;
     @LogLevel(LogLevel.Log4jLevel.Off)
-    String vncPassword;
+    private String vncPassword;
-    String vncAddr;
+    private String vncAddr;
-    Map<String, String> params;
+    private Map<String, String> details;
-    String uuid;
+    private Map<String, String> params;
-    String bootType;
+    private String uuid;
-    String bootMode;
+    private String bootType;
-    boolean enterHardwareSetup;
+    private String bootMode;
+    private boolean enterHardwareSetup;
 
-    DiskTO[] disks;
+    private DiskTO[] disks;
-    NicTO[] nics;
+    private NicTO[] nics;
-    GPUDeviceTO gpuDevice;
+    private GPUDeviceTO gpuDevice;
-    Integer vcpuMaxLimit;
+    private Integer vcpuMaxLimit;
-    List<String[]> vmData = null;
+    private List<String[]> vmData = null;
 
-    String configDriveLabel = null;
+    private String configDriveLabel = null;
-    String configDriveIsoRootFolder = null;
+    private String configDriveIsoRootFolder = null;
-    String configDriveIsoFile = null;
+    private String configDriveIsoFile = null;
-    NetworkElement.Location configDriveLocation = NetworkElement.Location.SECONDARY;
+    private NetworkElement.Location configDriveLocation = NetworkElement.Location.SECONDARY;
 
-    Double cpuQuotaPercentage = null;
+    private Double cpuQuotaPercentage = null;
 
-    Map<String, String> guestOsDetails = new HashMap<String, String>();
+    private Map<String, String> guestOsDetails = new HashMap<String, String>();
-    Map<String, String> extraConfig = new HashMap<>();
+    private Map<String, String> extraConfig = new HashMap<>();
-    Map<Long, String> networkIdToNetworkNameMap = new HashMap<>();
+    private Map<Long, String> networkIdToNetworkNameMap = new HashMap<>();
-    DeployAsIsInfoTO deployAsIsInfo;
+    private DeployAsIsInfoTO deployAsIsInfo;
-    String metadataManufacturer;
+    private String metadataManufacturer;
-    String metadataProductName;
+    private String metadataProductName;
+    private VirtualMachineMetadataTO metadata;
 
     public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader,
             String os, boolean enableHA, boolean limitCpuUse, String vncPassword) {
@@ -191,7 +195,11 @@ public class VirtualMachineTO {
         return maxSpeed;
     }
 
-    public boolean getLimitCpuUse() {
+    public boolean isEnableHA() {
+        return enableHA;
+    }
+
+    public boolean isLimitCpuUse() {
         return limitCpuUse;
     }
 
@@ -256,6 +264,10 @@ public class VirtualMachineTO {
         this.bootupScripts = bootupScripts;
     }
 
+    public void setEnableHA(boolean enableHA) {
+        this.enableHA = enableHA;
+    }
+
     public DiskTO[] getDisks() {
         return disks;
     }
@@ -289,11 +301,11 @@ public class VirtualMachineTO {
     }
 
     public Map<String, String> getDetails() {
-        return params;
+        return details;
     }
 
     public void setDetails(Map<String, String> params) {
-        this.params = params;
+        this.details = params;
     }
 
     public String getUuid() {
@@ -431,6 +443,42 @@ public class VirtualMachineTO {
         this.deployAsIsInfo = deployAsIsInfo;
     }
 
+    public void setSpeed(Integer speed) {
+        this.speed = speed;
+    }
+
+    public void setMinSpeed(Integer minSpeed) {
+        this.minSpeed = minSpeed;
+    }
+
+    public void setMaxSpeed(Integer maxSpeed) {
+        this.maxSpeed = maxSpeed;
+    }
+
+    public void setMinRam(long minRam) {
+        this.minRam = minRam;
+    }
+
+    public void setMaxRam(long maxRam) {
+        this.maxRam = maxRam;
+    }
+
+    public void setLimitCpuUse(boolean limitCpuUse) {
+        this.limitCpuUse = limitCpuUse;
+    }
+
+    public Map<String, String> getParams() {
+        return params;
+    }
+
+    public void setParams(Map<String, String> params) {
+        this.params = params;
+    }
+
+    public void setExtraConfig(Map<String, String> extraConfig) {
+        this.extraConfig = extraConfig;
+    }
+
     public String getMetadataManufacturer() {
         return metadataManufacturer;
     }
@@ -447,8 +495,28 @@ public class VirtualMachineTO {
         this.metadataProductName = metadataProductName;
     }
 
+    public VirtualMachineMetadataTO getMetadata() {
+        return metadata;
+    }
+
+    public void setMetadata(VirtualMachineMetadataTO metadata) {
+        this.metadata = metadata;
+    }
+
     @Override
     public String toString() {
         return String.format("VM {id: \"%s\", name: \"%s\", uuid: \"%s\", type: \"%s\"}", id, name, uuid, type);
     }
+
+    public Map<String, String> getExternalDetails() {
+        if (details == null) {
+            return new HashMap<>();
+        }
+        return details.entrySet().stream()
+                .filter(entry -> entry.getKey().startsWith(VmDetailConstants.EXTERNAL_DETAIL_PREFIX))
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey().substring(VmDetailConstants.EXTERNAL_DETAIL_PREFIX.length()),
+                        Map.Entry::getValue
+                ));
+    }
 }

api/src/main/java/com/cloud/api/commands/ListRecurringSnapshotScheduleCmd.java

@@ -35,10 +35,10 @@ public class ListRecurringSnapshotScheduleCmd extends BaseListCmd {
     //////////////// API parameters /////////////////////
     /////////////////////////////////////////////////////
 
-    @Parameter(name = ApiConstants.SNAPSHOT_POLICY_ID, type = CommandType.LONG, description = "lists recurring snapshots by snapshot policy ID")
+    @Parameter(name = ApiConstants.SNAPSHOT_POLICY_ID, type = CommandType.LONG, description = "Lists recurring Snapshots by Snapshot policy ID")
     private Long snapshotPolicyId;
 
-    @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.LONG, required = true, description = "list recurring snapshots by volume ID")
+    @Parameter(name = ApiConstants.VOLUME_ID, type = CommandType.LONG, required = true, description = "List recurring Snapshots by volume ID")
     private Long volumeId;
 
     /////////////////////////////////////////////////////

api/src/main/java/com/cloud/capacity/Capacity.java

@@ -34,13 +34,17 @@ public interface Capacity extends InternalIdentity, Identity {
     public static final short CAPACITY_TYPE_LOCAL_STORAGE = 9;
     public static final short CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET = 10;
     public static final short CAPACITY_TYPE_GPU = 19;
+    public static final short CAPACITY_TYPE_OBJECT_STORAGE = 20;
+    public static final short CAPACITY_TYPE_BACKUP_STORAGE = 21;
 
     public static final short CAPACITY_TYPE_CPU_CORE = 90;
 
     public static final List<Short> STORAGE_CAPACITY_TYPES = List.of(CAPACITY_TYPE_STORAGE,
             CAPACITY_TYPE_STORAGE_ALLOCATED,
             CAPACITY_TYPE_SECONDARY_STORAGE,
-            CAPACITY_TYPE_LOCAL_STORAGE);
+            CAPACITY_TYPE_LOCAL_STORAGE,
+            CAPACITY_TYPE_BACKUP_STORAGE,
+            CAPACITY_TYPE_OBJECT_STORAGE);
 
     public Long getHostOrPoolId();
 

api/src/main/java/com/cloud/configuration/ConfigurationService.java

@@ -17,7 +17,11 @@
 package com.cloud.configuration;
 
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
+import com.cloud.network.Network;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
 import org.apache.cloudstack.api.command.admin.network.CreateGuestNetworkIpv6PrefixCmd;
@@ -104,36 +108,22 @@ public interface ConfigurationService {
     /**
      * Updates a service offering
      *
-     * @param serviceOfferingId
-     * @param userId
-     * @param name
-     * @param displayText
-     * @param offerHA
-     * @param useVirtualNetwork
-     * @param tags
      * @return updated service offering
      */
     ServiceOffering updateServiceOffering(UpdateServiceOfferingCmd cmd);
 
     /**
      * Deletes a service offering
-     *
-     * @param userId
-     * @param serviceOfferingId
      */
     boolean deleteServiceOffering(DeleteServiceOfferingCmd cmd);
 
     /**
      * Retrieve ID of domains for a service offering
-     *
-     * @param serviceOfferingId
      */
     List<Long> getServiceOfferingDomains(Long serviceOfferingId);
 
     /**
      * Retrieve ID of domains for a service offering
-     *
-     * @param serviceOfferingId
      */
     List<Long> getServiceOfferingZones(Long serviceOfferingId);
 
@@ -143,7 +133,6 @@ public interface ConfigurationService {
      * @param cmd
      *            - the command specifying diskOfferingId, name, description, tags
      * @return updated disk offering
-     * @throws
      */
     DiskOffering updateDiskOffering(UpdateDiskOfferingCmd cmd);
 
@@ -153,34 +142,22 @@ public interface ConfigurationService {
      * @param cmd
      *            - the command specifying disk offering id
      * @return true or false
-     * @throws
      */
     boolean deleteDiskOffering(DeleteDiskOfferingCmd cmd);
 
     /**
      * Creates a new disk offering
-     *
-     * @param domainId
-     * @param name
-     * @param description
-     * @param numGibibytes
-     * @param mirrored
-     * @param size
      * @return ID
      */
     DiskOffering createDiskOffering(CreateDiskOfferingCmd cmd);
 
     /**
      * Retrieve ID of domains for a disk offering
-     *
-     * @param diskOfferingId
      */
     List<Long> getDiskOfferingDomains(Long diskOfferingId);
 
     /**
      * Retrieve ID of domains for a disk offering
-     *
-     * @param diskOfferingId
      */
     List<Long> getDiskOfferingZones(Long diskOfferingId);
 
@@ -201,11 +178,10 @@ public interface ConfigurationService {
      *            TODO
      * @param allocationState
      *            TODO
+     * @param storageAccessGroups
      * @return the new pod if successful, null otherwise
-     * @throws
-     * @throws
      */
-    Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState);
+    Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState, List<String> storageAccessGroups);
 
     /**
      * Creates a mutual exclusive IP range in the pod with same gateway, netmask.
@@ -223,8 +199,7 @@ public interface ConfigurationService {
     /**
      * Updates a mutually exclusive IP range in the pod.
      * @param cmd - The command specifying pod ID, current Start IP, current End IP, new Start IP, new End IP.
-     * @throws com.cloud.exception.ConcurrentOperationException
+     * @throws com.cloud.exception.ConcurrentOperationException when this pod is already being accessed
-     * @return Success
      */
     void updatePodIpRange(UpdatePodManagementNetworkIpRangeCmd cmd) throws ConcurrentOperationException;
 
@@ -245,9 +220,6 @@ public interface ConfigurationService {
 
     /**
      * Edits a pod in the database. Will not allow you to edit pods that are being used anywhere in the system.
-     *
-     * @param UpdatePodCmd
-     *            api command
      */
     Pod editPod(UpdatePodCmd cmd);
 
@@ -257,17 +229,12 @@ public interface ConfigurationService {
      * @param cmd
      *            - the command containing podId
      * @return true or false
-     * @throws ,
      */
     boolean deletePod(DeletePodCmd cmd);
 
     /**
      * Creates a new zone
-     *
-     * @param cmd
      * @return the zone if successful, null otherwise
-     * @throws
-     * @throws
      */
     DataCenter createZone(CreateZoneCmd cmd);
 
@@ -290,22 +257,7 @@ public interface ConfigurationService {
      * Adds a VLAN to the database, along with an IP address range. Can add three types of VLANs: (1) zone-wide VLANs on
      * the
      * virtual public network (2) pod-wide direct attached VLANs (3) account-specific direct attached VLANs
-     *
-     * @param userId
-     * @param vlanType
-     *            - either "DomR" (VLAN for a virtual public network) or "DirectAttached" (VLAN for IPs that will be
-     *            directly
-     *            attached to UserVMs)
-     * @param zoneId
-     * @param accountId
-     * @param podId
-     * @param add
-     * @param vlanId
-     * @param gateway
-     * @param startIP
-     * @param endIP
      * @throws ResourceAllocationException TODO
-     * @throws
      * @return The new Vlan object
      */
     Vlan createVlanAndPublicIpRange(CreateVlanIpRangeCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException,
@@ -320,9 +272,6 @@ public interface ConfigurationService {
     /**
      * Marks the account with the default zone-id.
      *
-     * @param accountName
-     * @param domainId
-     * @param defaultZoneId
      * @return The new account object
      */
     Account markDefaultZone(String accountName, long domainId, long defaultZoneId);
@@ -344,14 +293,12 @@ public interface ConfigurationService {
     /**
      * Retrieve ID of domains for a network offering
      *
-     * @param networkOfferingId
      */
     List<Long> getNetworkOfferingDomains(Long networkOfferingId);
 
     /**
      * Retrieve ID of domains for a network offering
      *
-     * @param networkOfferingId
      */
     List<Long> getNetworkOfferingZones(Long networkOfferingId);
 
@@ -372,4 +319,16 @@ public interface ConfigurationService {
     List<? extends PortableIp> listPortableIps(long id);
 
     Boolean isAccountAllowedToCreateOfferingsWithTags(IsAccountAllowedToCreateOfferingsWithTagsCmd cmd);
+
+    public static final Map<String, String> ProviderDetailKeyMap = Map.of(
+            Network.Provider.Nsx.getName(), ApiConstants.NSX_DETAIL_KEY,
+            Network.Provider.Netris.getName(), ApiConstants.NETRIS_DETAIL_KEY
+    );
+
+    public static boolean IsIpRangeForProvider(Network.Provider provider) {
+        if (Objects.isNull(provider)) {
+            return false;
+        }
+        return ProviderDetailKeyMap.containsKey(provider.getName());
+    }
 }

api/src/main/java/com/cloud/configuration/Resource.java

@@ -21,7 +21,7 @@ public interface Resource {
     short RESOURCE_UNLIMITED = -1;
     String UNLIMITED = "Unlimited";
 
-    enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage.
+    enum ResourceType { // All storage type resources are allocated_storage and not the physical storage.
         user_vm("user_vm", 0),
         public_ip("public_ip", 1),
         volume("volume", 2),
@@ -33,7 +33,12 @@ public interface Resource {
         cpu("cpu", 8),
         memory("memory", 9),
         primary_storage("primary_storage", 10),
-        secondary_storage("secondary_storage", 11);
+        secondary_storage("secondary_storage", 11),
+        backup("backup", 12),
+        backup_storage("backup_storage", 13),
+        bucket("bucket", 14),
+        object_storage("object_storage", 15),
+        gpu("gpu", 16);
 
         private String name;
         private int ordinal;
@@ -62,6 +67,10 @@ public interface Resource {
             }
             return null;
         }
+
+        public static Boolean isStorageType(ResourceType type) {
+            return (type == primary_storage || type == secondary_storage || type == backup_storage || type == object_storage);
+        }
     }
 
     public static class ResourceOwnerType {

api/src/main/java/com/cloud/cpu/CPU.java

@@ -22,7 +22,8 @@ public class CPU {
     public enum CPUArch {
         x86("i686", 32),
         amd64("x86_64", 64),
-        arm64("aarch64", 64);
+        arm64("aarch64", 64),
+        s390x("s390x", 64);
 
         private final String type;
         private final int bits;

api/src/main/java/com/cloud/dc/Pod.java

@@ -43,4 +43,6 @@ public interface Pod extends InfrastructureEntity, Grouping, Identity, InternalI
     AllocationState getAllocationState();
 
     boolean getExternalDhcp();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java

@@ -62,11 +62,11 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner {
             "vm.allocation.algorithm",
             "Advanced",
             "random",
-            "Order in which hosts within a cluster will be considered for VM/volume allocation. The value can be 'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', or 'firstfitleastconsumed'.",
+            "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', 'firstfit', 'userdispersing', or 'firstfitleastconsumed'.",
             true,
             ConfigKey.Scope.Global, null, null, null, null, null,
             ConfigKey.Kind.Select,
-            "random,firstfit,userdispersing,userconcentratedpod_random,userconcentratedpod_firstfit,firstfitleastconsumed");
+            "random,firstfit,userdispersing,firstfitleastconsumed");
 
     /**
      * This is called to determine list of possible clusters where a virtual

api/src/main/java/com/cloud/deploy/DeploymentPlanner.java

@@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter {
     boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid);
 
     public enum AllocationAlgorithm {
-        random, firstfit, userdispersing, userconcentratedpod_random, userconcentratedpod_firstfit;
+        random, firstfit, userdispersing;
     }
 
     public enum PlannerResourceUsage {

api/src/main/java/com/cloud/event/EventTypes.java

@@ -27,15 +27,21 @@ import org.apache.cloudstack.api.response.ClusterResponse;
 import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
+import org.apache.cloudstack.backup.BackupRepositoryService;
 import org.apache.cloudstack.config.Configuration;
 import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet;
+import org.apache.cloudstack.extension.Extension;
+import org.apache.cloudstack.extension.ExtensionCustomAction;
+import org.apache.cloudstack.gpu.GpuCard;
+import org.apache.cloudstack.gpu.GpuDevice;
+import org.apache.cloudstack.gpu.VgpuProfile;
 import org.apache.cloudstack.ha.HAConfig;
 import org.apache.cloudstack.network.BgpPeer;
 import org.apache.cloudstack.network.Ipv4GuestSubnetNetworkMap;
 import org.apache.cloudstack.quota.QuotaTariff;
-import org.apache.cloudstack.storage.sharedfs.SharedFS;
 import org.apache.cloudstack.storage.object.Bucket;
 import org.apache.cloudstack.storage.object.ObjectStore;
+import org.apache.cloudstack.storage.sharedfs.SharedFS;
 import org.apache.cloudstack.usage.Usage;
 import org.apache.cloudstack.vm.schedule.VMSchedule;
 
@@ -289,6 +295,8 @@ public class EventTypes {
 
     //registering userdata events
     public static final String EVENT_REGISTER_USER_DATA = "REGISTER.USER.DATA";
+    public static final String EVENT_REGISTER_CNI_CONFIG = "REGISTER.CNI.CONFIG";
+    public static final String EVENT_DELETE_CNI_CONFIG = "DELETE.CNI.CONFIG";
 
     //register for user API and secret keys
     public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY";
@@ -374,6 +382,21 @@ public class EventTypes {
     public static final String EVENT_DISK_OFFERING_EDIT = "DISK.OFFERING.EDIT";
     public static final String EVENT_DISK_OFFERING_DELETE = "DISK.OFFERING.DELETE";
 
+    // GPU Cards
+    public static final String EVENT_GPU_CARD_CREATE = "GPU.CARD.CREATE";
+    public static final String EVENT_GPU_CARD_EDIT = "GPU.CARD.EDIT";
+    public static final String EVENT_GPU_CARD_DELETE = "GPU.CARD.DELETE";
+
+    // vGPU Profile
+    public static final String EVENT_VGPU_PROFILE_CREATE = "VGPU.PROFILE.CREATE";
+    public static final String EVENT_VGPU_PROFILE_EDIT = "VGPU.PROFILE.EDIT";
+    public static final String EVENT_VGPU_PROFILE_DELETE = "VGPU.PROFILE.DELETE";
+
+    // GPU Devices
+    public static final String EVENT_GPU_DEVICE_CREATE = "GPU.DEVICE.CREATE";
+    public static final String EVENT_GPU_DEVICE_EDIT = "GPU.DEVICE.EDIT";
+    public static final String EVENT_GPU_DEVICE_DELETE = "GPU.DEVICE.DELETE";
+
     // Network offerings
     public static final String EVENT_NETWORK_OFFERING_CREATE = "NETWORK.OFFERING.CREATE";
     public static final String EVENT_NETWORK_OFFERING_ASSIGN = "NETWORK.OFFERING.ASSIGN";
@@ -465,6 +488,7 @@ public class EventTypes {
     public static final String EVENT_ENABLE_PRIMARY_STORAGE = "ENABLE.PS";
     public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS";
     public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL";
+    public static final String EVENT_CONFIGURE_STORAGE_ACCESS = "CONFIGURE.STORAGE.ACCESS";
     public static final String EVENT_CHANGE_STORAGE_POOL_SCOPE = "CHANGE.STORAGE.POOL.SCOPE";
 
     // VPN
@@ -479,6 +503,7 @@ public class EventTypes {
     public static final String EVENT_S2S_VPN_CUSTOMER_GATEWAY_CREATE = "VPN.S2S.CUSTOMER.GATEWAY.CREATE";
     public static final String EVENT_S2S_VPN_CUSTOMER_GATEWAY_DELETE = "VPN.S2S.CUSTOMER.GATEWAY.DELETE";
     public static final String EVENT_S2S_VPN_CUSTOMER_GATEWAY_UPDATE = "VPN.S2S.CUSTOMER.GATEWAY.UPDATE";
+    public static final String EVENT_S2S_VPN_GATEWAY_OBSOLETE_PARAMS = "VPN.S2S.GATEWAY.OBSOLETE.PARAMS";
     public static final String EVENT_S2S_VPN_CONNECTION_CREATE = "VPN.S2S.CONNECTION.CREATE";
     public static final String EVENT_S2S_VPN_CONNECTION_DELETE = "VPN.S2S.CONNECTION.DELETE";
     public static final String EVENT_S2S_VPN_CONNECTION_RESET = "VPN.S2S.CONNECTION.RESET";
@@ -496,6 +521,8 @@ public class EventTypes {
 
     public static final String EVENT_ZONE_VLAN_ASSIGN = "ZONE.VLAN.ASSIGN";
     public static final String EVENT_ZONE_VLAN_RELEASE = "ZONE.VLAN.RELEASE";
+    public static final String EVENT_ZONE_VXLAN_ASSIGN = "ZONE.VXLAN.ASSIGN";
+    public static final String EVENT_ZONE_VXLAN_RELEASE = "ZONE.VXLAN.RELEASE";
 
     // Projects
     public static final String EVENT_PROJECT_CREATE = "PROJECT.CREATE";
@@ -607,11 +634,13 @@ public class EventTypes {
     public static final String EVENT_VM_BACKUP_CREATE = "BACKUP.CREATE";
     public static final String EVENT_VM_BACKUP_RESTORE = "BACKUP.RESTORE";
     public static final String EVENT_VM_BACKUP_DELETE = "BACKUP.DELETE";
+    public static final String EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED = "BACKUP.OFFERING.BACKUPS.DEL";
     public static final String EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM = "BACKUP.RESTORE.VOLUME.TO.VM";
     public static final String EVENT_VM_BACKUP_SCHEDULE_CONFIGURE = "BACKUP.SCHEDULE.CONFIGURE";
     public static final String EVENT_VM_BACKUP_SCHEDULE_DELETE = "BACKUP.SCHEDULE.DELETE";
     public static final String EVENT_VM_BACKUP_USAGE_METRIC = "BACKUP.USAGE.METRIC";
     public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT";
+    public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP";
 
     // external network device events
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD";
@@ -687,6 +716,9 @@ public class EventTypes {
     public static final String EVENT_EXTERNAL_OPENDAYLIGHT_CONFIGURE_CONTROLLER = "PHYSICAL.ODLCONTROLLER.CONFIGURE";
 
     //Guest OS related events
+    public static final String EVENT_GUEST_OS_CATEGORY_ADD = "GUEST.OS.CATEGORY.ADD";
+    public static final String EVENT_GUEST_OS_CATEGORY_DELETE = "GUEST.OS.CATEGORY.DELETE";
+    public static final String EVENT_GUEST_OS_CATEGORY_UPDATE = "GUEST.OS.CATEGORY.UPDATE";
     public static final String EVENT_GUEST_OS_ADD = "GUEST.OS.ADD";
     public static final String EVENT_GUEST_OS_REMOVE = "GUEST.OS.REMOVE";
     public static final String EVENT_GUEST_OS_UPDATE = "GUEST.OS.UPDATE";
@@ -739,6 +771,13 @@ public class EventTypes {
     //Purge resources
     public static final String EVENT_PURGE_EXPUNGED_RESOURCES = "PURGE.EXPUNGED.RESOURCES";
 
+    // Management Server
+    public static final String EVENT_MS_MAINTENANCE_PREPARE = "MS.MAINTENANCE.PREPARE";
+    public static final String EVENT_MS_MAINTENANCE_CANCEL = "MS.MAINTENANCE.CANCEL";
+    public static final String EVENT_MS_SHUTDOWN_PREPARE = "MS.SHUTDOWN.PREPARE";
+    public static final String EVENT_MS_SHUTDOWN_CANCEL = "MS.SHUTDOWN.CANCEL";
+    public static final String EVENT_MS_SHUTDOWN = "MS.SHUTDOWN";
+
     // OBJECT STORE
     public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE";
     public static final String EVENT_OBJECT_STORE_DELETE = "OBJECT.STORE.DELETE";
@@ -785,6 +824,40 @@ public class EventTypes {
     public static final String EVENT_SHAREDFS_EXPUNGE = "SHAREDFS.EXPUNGE";
     public static final String EVENT_SHAREDFS_RECOVER = "SHAREDFS.RECOVER";
 
+    // Resource Limit
+    public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE";
+
+    // Management Server
+    public static final String EVENT_MANAGEMENT_SERVER_REMOVE = "MANAGEMENT.SERVER.REMOVE";
+
+    // VM Lease
+    public static final String VM_LEASE_EXPIRED = "VM.LEASE.EXPIRED";
+    public static final String VM_LEASE_DISABLED = "VM.LEASE.DISABLED";
+    public static final String VM_LEASE_CANCELLED = "VM.LEASE.CANCELLED";
+    public static final String VM_LEASE_EXPIRING = "VM.LEASE.EXPIRING";
+
+    // GUI Theme
+    public static final String EVENT_GUI_THEME_CREATE = "GUI.THEME.CREATE";
+    public static final String EVENT_GUI_THEME_REMOVE = "GUI.THEME.REMOVE";
+    public static final String EVENT_GUI_THEME_UPDATE = "GUI.THEME.UPDATE";
+
+    // Extension
+    public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE";
+    public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE";
+    public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE";
+    public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER";
+    public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_UPDATE = "EXTENSION.CUSTOM.ACTION.UPDATE";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_DELETE = "EXTENSION.CUSTOM.ACTION.DELETE";
+
+    // Custom Action
+    public static final String EVENT_CUSTOM_ACTION = "CUSTOM.ACTION";
+
+    // Backup Repository
+    public static final String EVENT_BACKUP_REPOSITORY_ADD = "BACKUP.REPOSITORY.ADD";
+    public static final String EVENT_BACKUP_REPOSITORY_UPDATE = "BACKUP.REPOSITORY.UPDATE";
+
     static {
 
         // TODO: need a way to force author adding event types to declare the entity details as well, with out braking
@@ -979,6 +1052,21 @@ public class EventTypes {
         entityEventDetails.put(EVENT_DISK_OFFERING_EDIT, DiskOffering.class);
         entityEventDetails.put(EVENT_DISK_OFFERING_DELETE, DiskOffering.class);
 
+        // GPU Cards
+        entityEventDetails.put(EVENT_GPU_CARD_CREATE, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_EDIT, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_DELETE, GpuCard.class);
+
+        // vGPU Profiles
+        entityEventDetails.put(EVENT_VGPU_PROFILE_CREATE, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_EDIT, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_DELETE, VgpuProfile.class);
+
+        // GPU Devices
+        entityEventDetails.put(EVENT_GPU_DEVICE_CREATE, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_EDIT, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_DELETE, GpuDevice.class);
+
         // Network offerings
         entityEventDetails.put(EVENT_NETWORK_OFFERING_CREATE, NetworkOffering.class);
         entityEventDetails.put(EVENT_NETWORK_OFFERING_ASSIGN, NetworkOffering.class);
@@ -1064,6 +1152,7 @@ public class EventTypes {
         entityEventDetails.put(EVENT_S2S_VPN_CUSTOMER_GATEWAY_CREATE, Site2SiteCustomerGateway.class);
         entityEventDetails.put(EVENT_S2S_VPN_CUSTOMER_GATEWAY_DELETE, Site2SiteCustomerGateway.class);
         entityEventDetails.put(EVENT_S2S_VPN_CUSTOMER_GATEWAY_UPDATE, Site2SiteCustomerGateway.class);
+        entityEventDetails.put(EVENT_S2S_VPN_GATEWAY_OBSOLETE_PARAMS, Site2SiteCustomerGateway.class);
         entityEventDetails.put(EVENT_S2S_VPN_CONNECTION_CREATE, Site2SiteVpnConnection.class);
         entityEventDetails.put(EVENT_S2S_VPN_CONNECTION_DELETE, Site2SiteVpnConnection.class);
         entityEventDetails.put(EVENT_S2S_VPN_CONNECTION_RESET, Site2SiteVpnConnection.class);
@@ -1230,6 +1319,12 @@ public class EventTypes {
         entityEventDetails.put(EVENT_UPDATE_IMAGE_STORE_ACCESS_STATE, ImageStore.class);
         entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs");
 
+        entityEventDetails.put(EVENT_MS_MAINTENANCE_PREPARE, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_MAINTENANCE_CANCEL, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_SHUTDOWN_PREPARE, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_SHUTDOWN_CANCEL, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_SHUTDOWN, "ManagementServer");
+
         //Object Store
         entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class);
         entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class);
@@ -1273,6 +1368,34 @@ public class EventTypes {
         entityEventDetails.put(EVENT_SHAREDFS_DESTROY, SharedFS.class);
         entityEventDetails.put(EVENT_SHAREDFS_EXPUNGE, SharedFS.class);
         entityEventDetails.put(EVENT_SHAREDFS_RECOVER, SharedFS.class);
+
+        // Management Server
+        entityEventDetails.put(EVENT_MANAGEMENT_SERVER_REMOVE, "ManagementServer");
+
+        // VM Lease
+        entityEventDetails.put(VM_LEASE_EXPIRED, VirtualMachine.class);
+        entityEventDetails.put(VM_LEASE_EXPIRING, VirtualMachine.class);
+        entityEventDetails.put(VM_LEASE_DISABLED, VirtualMachine.class);
+        entityEventDetails.put(VM_LEASE_CANCELLED, VirtualMachine.class);
+
+        // GUI theme
+        entityEventDetails.put(EVENT_GUI_THEME_CREATE, "GuiTheme");
+        entityEventDetails.put(EVENT_GUI_THEME_REMOVE, "GuiTheme");
+        entityEventDetails.put(EVENT_GUI_THEME_UPDATE, "GuiTheme");
+
+        // Extension
+        entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_UPDATE, ExtensionCustomAction.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_DELETE, ExtensionCustomAction.class);
+
+        // Backup Repository
+        entityEventDetails.put(EVENT_BACKUP_REPOSITORY_ADD, BackupRepositoryService.class);
+        entityEventDetails.put(EVENT_BACKUP_REPOSITORY_UPDATE, BackupRepositoryService.class);
     }
 
     public static boolean isNetworkEvent(String eventType) {

api/src/main/java/com/cloud/exception/OperationTimedoutException.java

@@ -40,7 +40,7 @@ public class OperationTimedoutException extends CloudException {
     boolean _isActive;
 
     public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) {
-        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time);
+        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs");
         _agentId = agentId;
         _seqId = seqId;
         _time = time;

api/src/main/java/com/cloud/host/Host.java

@@ -53,9 +53,12 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
             return strs;
         }
     }
-    public static final String HOST_UEFI_ENABLE = "host.uefi.enable";
+
-    public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+    String HOST_UEFI_ENABLE = "host.uefi.enable";
-    public static final String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
+    String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+    String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
+    String HOST_OVFTOOL_VERSION = "host.ovftool.version";
+    String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
 
     /**
      * @return name of the machine.
@@ -177,6 +180,8 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
      */
     Long getManagementServerId();
 
+    Long getLastManagementServerId();
+
     /*
      *@return removal date
      */
@@ -211,4 +216,6 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
     ResourceState getResourceState();
 
     CPU.CPUArch getArch();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/host/HostStats.java

@@ -36,5 +36,4 @@ public interface HostStats {
     public HostStats getHostStats();
 
     public double getLoadAverage();
-    // public double getXapiMemoryUsageKBs();
 }

api/src/main/java/com/cloud/host/Status.java

@@ -127,6 +127,7 @@ public enum Status {
         s_fsm.addTransition(Status.Connecting, Event.HostDown, Status.Down);
         s_fsm.addTransition(Status.Connecting, Event.Ping, Status.Connecting);
         s_fsm.addTransition(Status.Connecting, Event.ManagementServerDown, Status.Disconnected);
+        s_fsm.addTransition(Status.Connecting, Event.StartAgentRebalance, Status.Rebalancing);
         s_fsm.addTransition(Status.Connecting, Event.AgentDisconnected, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.PingTimeout, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.AgentDisconnected, Status.Alert);

api/src/main/java/com/cloud/hypervisor/Hypervisor.java

@@ -31,20 +31,22 @@ import java.util.stream.Collectors;
 import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate;
 import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.RootDiskSizeOverride;
 import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.VmStorageMigration;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.VmStorageMigrationWithSnapshots;
 
 public class Hypervisor {
     public static class HypervisorType {
         public enum Functionality {
             DirectDownloadTemplate,
             RootDiskSizeOverride,
-            VmStorageMigration
+            VmStorageMigration,
+            VmStorageMigrationWithSnapshots
         }
 
         private static final Map<String, HypervisorType> hypervisorTypeMap = new LinkedHashMap<>();
         public static final HypervisorType None = new HypervisorType("None"); //for storage hosts
         public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
         public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2, EnumSet.of(DirectDownloadTemplate, RootDiskSizeOverride, VmStorageMigration));
-        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA, EnumSet.of(RootDiskSizeOverride, VmStorageMigration, VmStorageMigrationWithSnapshots));
         public static final HypervisorType Hyperv = new HypervisorType("Hyperv");
         public static final HypervisorType VirtualBox = new HypervisorType("VirtualBox");
         public static final HypervisorType Parralels = new HypervisorType("Parralels");
@@ -54,6 +56,7 @@ public class Hypervisor {
         public static final HypervisorType Ovm3 = new HypervisorType("Ovm3", ImageFormat.RAW);
         public static final HypervisorType LXC = new HypervisorType("LXC");
         public static final HypervisorType Custom = new HypervisorType("Custom", null, EnumSet.of(RootDiskSizeOverride));
+        public static final HypervisorType External = new HypervisorType("External", null, EnumSet.of(RootDiskSizeOverride));
         public static final HypervisorType Any = new HypervisorType("Any"); /*If you don't care about the hypervisor type*/
         private final String name;
         private final ImageFormat imageFormat;

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java

@@ -44,6 +44,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
         AutoscaleRequested,
         ScaleUpRequested,
         ScaleDownRequested,
+        AddNodeRequested,
+        RemoveNodeRequested,
         UpgradeRequested,
         OperationSucceeded,
         OperationFailed,
@@ -58,7 +60,10 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
         Stopping("Resources for the Kubernetes cluster are being destroyed"),
         Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"),
         Scaling("Transient state in which resources are either getting scaled up/down"),
+        ScalingStoppedCluster("Transient state in which the service offerings of stopped clusters are getting scaled"),
         Upgrading("Transient state in which cluster is getting upgraded"),
+        Importing("Transient state in which additional nodes are added as worker nodes to a cluster"),
+        RemovingNodes("Transient state in which additional nodes are removed from a cluster"),
         Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."),
         Recovering("State in which Kubernetes cluster is recovering from alert state"),
         Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"),
@@ -83,19 +88,35 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
             s_fsm.addTransition(State.Stopping, Event.OperationFailed, State.Alert);
 
             s_fsm.addTransition(State.Stopped, Event.StartRequested, State.Starting);
+            s_fsm.addTransition(State.Stopped, Event.OperationSucceeded, State.Stopped);
+            s_fsm.addTransition(State.Running, Event.OperationSucceeded, State.Running);
 
             s_fsm.addTransition(State.Running, Event.FaultsDetected, State.Alert);
 
             s_fsm.addTransition(State.Running, Event.AutoscaleRequested, State.Scaling);
             s_fsm.addTransition(State.Running, Event.ScaleUpRequested, State.Scaling);
             s_fsm.addTransition(State.Running, Event.ScaleDownRequested, State.Scaling);
+            s_fsm.addTransition(State.Stopped, Event.ScaleUpRequested, State.ScalingStoppedCluster);
             s_fsm.addTransition(State.Scaling, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Scaling, Event.OperationFailed, State.Alert);
+            s_fsm.addTransition(State.ScalingStoppedCluster, Event.OperationSucceeded, State.Stopped);
+            s_fsm.addTransition(State.ScalingStoppedCluster, Event.OperationFailed, State.Alert);
 
             s_fsm.addTransition(State.Running, Event.UpgradeRequested, State.Upgrading);
             s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert);
 
+            s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing);
+            s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing);
+            s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.Importing, Event.OperationFailed, State.Running);
+            s_fsm.addTransition(State.Alert, Event.OperationSucceeded, State.Running);
+
+            s_fsm.addTransition(State.Running, Event.RemoveNodeRequested, State.RemovingNodes);
+            s_fsm.addTransition(State.Alert, Event.RemoveNodeRequested, State.RemovingNodes);
+            s_fsm.addTransition(State.RemovingNodes, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.RemovingNodes, Event.OperationFailed, State.Running);
+
             s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering);
             s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert);
@@ -142,4 +163,14 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
     Long getMaxSize();
     Long getSecurityGroupId();
     ClusterType getClusterType();
+    Long getControlNodeServiceOfferingId();
+    Long getWorkerNodeServiceOfferingId();
+    Long getEtcdNodeServiceOfferingId();
+    Long getControlNodeTemplateId();
+    Long getWorkerNodeTemplateId();
+    Long getEtcdNodeTemplateId();
+    Long getEtcdNodeCount();
+    Long getCniConfigId();
+    String getCniConfigDetails();
+    boolean isCsiEnabled();
 }

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -18,14 +18,23 @@ package com.cloud.kubernetes.cluster;
 
 import org.apache.cloudstack.acl.ControlledEntity;
 
+import java.util.Map;
+
 import com.cloud.user.Account;
 import com.cloud.uservm.UserVm;
 import com.cloud.utils.component.Adapter;
 
 public interface KubernetesServiceHelper extends Adapter {
 
+    enum KubernetesClusterNodeType {
+        CONTROL, WORKER, ETCD, DEFAULT
+    }
+
     ControlledEntity findByUuid(String uuid);
     ControlledEntity findByVmId(long vmId);
     void checkVmCanBeDestroyed(UserVm userVm);
+    boolean isValidNodeType(String nodeType);
+    Map<String, Long> getServiceOfferingNodeTypeMap(Map<String, Map<String, String>> serviceOfferingNodeTypeMap);
+    Map<String, Long> getTemplateNodeTypeMap(Map<String, Map<String, String>> templateNodeTypeMap);
     void cleanupForAccount(Account account);
 }

api/src/main/java/com/cloud/network/IpAddress.java

@@ -99,4 +99,5 @@ public interface IpAddress extends ControlledEntity, Identity, InternalIdentity,
 
     boolean isForSystemVms();
 
+    boolean isForRouter();
 }

api/src/main/java/com/cloud/network/Ipv6Service.java

@@ -45,7 +45,7 @@ public interface Ipv6Service extends PluggableService, Configurable {
     static final ConfigKey<Boolean> Ipv6OfferingCreationEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
             "ipv6.offering.enabled",
             "false",
-            "Indicates whether creation of IPv6 network/VPC offering is enabled or not.",
+            "Indicates whether creation of IPv6 Network/VPC offering is enabled or not.",
             true);
 
     static final ConfigKey<Integer> Ipv6PrefixSubnetCleanupInterval = new ConfigKey<Integer>("Advanced", Integer.class,

api/src/main/java/com/cloud/network/Network.java

@@ -206,6 +206,7 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
         public static final Provider Tungsten = new Provider("Tungsten", false);
 
         public static final Provider Nsx = new Provider("Nsx", false);
+        public static final Provider Netris = new Provider("Netris", false);
 
         private final String name;
         private final boolean isExternal;
@@ -324,9 +325,9 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
 
     public enum State {
 
-        Allocated("Indicates the network configuration is in allocated but not setup"), Setup("Indicates the network configuration is setup"), Implementing(
+        Allocated("Indicates the Network configuration is in allocated but not setup"), Setup("Indicates the Network configuration is setup"), Implementing(
-                "Indicates the network configuration is being implemented"), Implemented("Indicates the network configuration is in use"), Shutdown(
+                "Indicates the Network configuration is being implemented"), Implemented("Indicates the Network configuration is in use"), Shutdown(
-                "Indicates the network configuration is being destroyed"), Destroy("Indicates that the network is destroyed");
+                "Indicates the Network configuration is being destroyed"), Destroy("Indicates that the Network is destroyed");
 
         protected static final StateMachine2<State, Network.Event, Network> s_fsm = new StateMachine2<State, Network.Event, Network>();
 

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -125,6 +125,10 @@ public interface NetworkModel {
      */
     String getNextAvailableMacAddressInNetwork(long networkConfigurationId) throws InsufficientAddressCapacityException;
 
+    String getUniqueMacAddress(long macAddress, long networkId, long datacenterId) throws InsufficientAddressCapacityException;
+
+    boolean isMACUnique(String mac, long networkId);
+
     PublicIpAddress getPublicIpAddress(long ipAddressId);
 
     List<? extends Vlan> listPodVlans(long podId);
@@ -305,6 +309,8 @@ public interface NetworkModel {
 
     NicProfile getNicProfile(VirtualMachine vm, long networkId, String broadcastUri);
 
+    NicProfile getNicProfile(VirtualMachine vm, Nic nic, DataCenter dataCenter);
+
     Set<Long> getAvailableIps(Network network, String requestedIp);
 
     String getDomainNetworkDomain(long domainId, long zoneId);
@@ -362,4 +368,8 @@ public interface NetworkModel {
 
     boolean checkSecurityGroupSupportForNetwork(Account account, DataCenter zone, List<Long> networkIds,
                                                 List<Long> securityGroupsIds);
+
+    default long getMacIdentifier(Long dataCenterId) {
+        return 0;
+    }
 }

api/src/main/java/com/cloud/network/NetworkService.java

@@ -19,7 +19,6 @@ package com.cloud.network;
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.dc.DataCenter;
 import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin;
 import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd;
@@ -39,13 +38,16 @@ import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd;
 import org.apache.cloudstack.api.command.user.vm.ListNicsCmd;
 import org.apache.cloudstack.api.response.AcquirePodIpCmdResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.dc.DataCenter;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.network.Network.IpAddresses;
 import com.cloud.network.Network.Service;
 import com.cloud.network.Networks.TrafficType;
@@ -57,7 +59,6 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicSecondaryIp;
-import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 
 /**
  * The NetworkService interface is the "public" api to entities that make requests to the orchestration engine
@@ -80,7 +81,7 @@ public interface NetworkService {
             true, ConfigKey.Scope.Zone);
 
     public static final ConfigKey<Boolean> AllowUsersToSpecifyVRMtu = new ConfigKey<>("Advanced", Boolean.class,
-            "allow.end.users.to.specify.vr.mtu", "false", "Allow end users to specify VR MTU",
+            "allow.end.users.to.specify.vr.mtu", "false", "Allow end Users to specify VR MTU",
             true, ConfigKey.Scope.Zone);
 
     List<? extends Network> getIsolatedNetworksOwnedByAccountInZone(long zoneId, Account owner);
@@ -268,4 +269,10 @@ public interface NetworkService {
     InternalLoadBalancerElementService getInternalLoadBalancerElementByNetworkServiceProviderId(long networkProviderId);
     InternalLoadBalancerElementService getInternalLoadBalancerElementById(long providerId);
     List<InternalLoadBalancerElementService> getInternalLoadBalancerElements();
+
+    boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException;
+
+    IpAddresses getIpAddressesFromIps(String ipAddress, String ip6Address, String macAddress);
+
+    String getNicVlanValueForExternalVm(NicTO nic);
 }

api/src/main/java/com/cloud/network/Networks.java

@@ -78,7 +78,7 @@ public class Networks {
             }
             @Override
             public String getValueFrom(URI uri) {
-                return uri.getAuthority();
+                return uri == null ? null : uri.getAuthority();
             }
         },
         Vswitch("vs", String.class), LinkLocal(null, null), Vnet("vnet", Long.class), Storage("storage", Integer.class), Lswitch("lswitch", String.class) {
@@ -96,7 +96,7 @@ public class Networks {
              */
             @Override
             public String getValueFrom(URI uri) {
-                return uri.getSchemeSpecificPart();
+                return uri == null ? null : uri.getSchemeSpecificPart();
             }
         },
         Mido("mido", String.class), Pvlan("pvlan", String.class),
@@ -129,7 +129,8 @@ public class Networks {
         UnDecided(null, null),
         OpenDaylight("opendaylight", String.class),
         TUNGSTEN("tf", String.class),
-        NSX("nsx", String.class);
+        NSX("nsx", String.class),
+        Netris("netris", String.class);
 
         private final String scheme;
         private final Class<?> type;
@@ -176,7 +177,7 @@ public class Networks {
          * @return the scheme as BroadcastDomainType
          */
         public static BroadcastDomainType getSchemeValue(URI uri) {
-            return toEnumValue(uri.getScheme());
+            return toEnumValue(uri == null ? null : uri.getScheme());
         }
 
         /**
@@ -190,7 +191,7 @@ public class Networks {
             if (com.cloud.dc.Vlan.UNTAGGED.equalsIgnoreCase(str)) {
                 return Native;
             }
-            return getSchemeValue(new URI(str));
+            return getSchemeValue(str == null ? null : new URI(str));
         }
 
         /**
@@ -219,7 +220,7 @@ public class Networks {
          * @return the host part as String
          */
         public String getValueFrom(URI uri) {
-            return uri.getHost();
+            return uri == null ? null : uri.getHost();
         }
 
         /**
@@ -242,7 +243,7 @@ public class Networks {
          * @throws URISyntaxException the string is not even an uri
          */
         public static String getValue(String uriString) throws URISyntaxException {
-            return getValue(new URI(uriString));
+            return getValue(uriString == null ? null : new URI(uriString));
         }
 
         /**

api/src/main/java/com/cloud/network/PhysicalNetworkTrafficType.java

@@ -41,4 +41,6 @@ public interface PhysicalNetworkTrafficType extends InternalIdentity, Identity {
     String getHypervNetworkLabel();
 
     String getOvm3NetworkLabel();
+
+    String getVlan();
 }

api/src/main/java/com/cloud/network/RouterHealthCheckResult.java

@@ -26,7 +26,7 @@ public interface RouterHealthCheckResult {
 
     String getCheckType();
 
-    boolean getCheckResult();
+    VirtualNetworkApplianceService.RouterHealthStatus getCheckResult();
 
     Date getLastUpdateTime();
 

api/src/main/java/com/cloud/network/SDNProviderNetworkRule.java

@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network;
+
+import java.util.List;
+
+public class SDNProviderNetworkRule {
+
+    protected long domainId;
+    protected long accountId;
+    protected long zoneId;
+    protected Long networkResourceId;
+    protected String networkResourceName;
+    protected boolean isVpcResource;
+    protected long vmId;
+    protected long ruleId;
+    protected String publicIp;
+    protected String vmIp;
+    protected String publicPort;
+    protected String privatePort;
+    protected String protocol;
+    protected String algorithm;
+    protected List<String> sourceCidrList;
+    protected List<String> destinationCidrList;
+    protected Integer icmpCode;
+
+    protected Integer icmpType;
+    protected String trafficType;
+    protected Network.Service service;
+
+    public long getDomainId() {
+        return domainId;
+    }
+
+    public void setDomainId(long domainId) {
+        this.domainId = domainId;
+    }
+
+    public long getAccountId() {
+        return accountId;
+    }
+
+    public void setAccountId(long accountId) {
+        this.accountId = accountId;
+    }
+
+    public long getZoneId() {
+        return zoneId;
+    }
+
+    public void setZoneId(long zoneId) {
+        this.zoneId = zoneId;
+    }
+
+    public Long getNetworkResourceId() {
+        return networkResourceId;
+    }
+
+    public void setNetworkResourceId(Long networkResourceId) {
+        this.networkResourceId = networkResourceId;
+    }
+
+    public String getNetworkResourceName() {
+        return networkResourceName;
+    }
+
+    public void setNetworkResourceName(String networkResourceName) {
+        this.networkResourceName = networkResourceName;
+    }
+
+    public boolean isVpcResource() {
+        return isVpcResource;
+    }
+
+    public void setVpcResource(boolean vpcResource) {
+        isVpcResource = vpcResource;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public void setVmId(long vmId) {
+        this.vmId = vmId;
+    }
+
+    public long getRuleId() {
+        return ruleId;
+    }
+
+    public void setRuleId(long ruleId) {
+        this.ruleId = ruleId;
+    }
+
+    public String getPublicIp() {
+        return publicIp;
+    }
+
+    public void setPublicIp(String publicIp) {
+        this.publicIp = publicIp;
+    }
+
+    public String getVmIp() {
+        return vmIp;
+    }
+
+    public void setVmIp(String vmIp) {
+        this.vmIp = vmIp;
+    }
+
+    public String getPublicPort() {
+        return publicPort;
+    }
+
+    public void setPublicPort(String publicPort) {
+        this.publicPort = publicPort;
+    }
+
+    public String getPrivatePort() {
+        return privatePort;
+    }
+
+    public void setPrivatePort(String privatePort) {
+        this.privatePort = privatePort;
+    }
+
+    public String getProtocol() {
+        return protocol;
+    }
+
+    public void setProtocol(String protocol) {
+        this.protocol = protocol;
+    }
+
+    public void setAlgorithm(String algorithm) {
+        this.algorithm = algorithm;
+    }
+
+    public String getAlgorithm() {
+        return algorithm;
+    }
+
+    public Network.Service getService() {
+        return service;
+    }
+
+    public void setService(Network.Service service) {
+        this.service = service;
+    }
+
+    public Integer getIcmpCode() {
+        return icmpCode;
+    }
+
+    public void setIcmpCode(Integer icmpCode) {
+        this.icmpCode = icmpCode;
+    }
+
+    public Integer getIcmpType() {
+        return icmpType;
+    }
+
+    public void setIcmpType(Integer icmpType) {
+        this.icmpType = icmpType;
+    }
+
+    public List<String> getSourceCidrList() {
+        return sourceCidrList;
+    }
+
+    public void setSourceCidrList(List<String> sourceCidrList) {
+        this.sourceCidrList = sourceCidrList;
+    }
+
+    public List<String> getDestinationCidrList() {
+        return destinationCidrList;
+    }
+
+    public void setDestinationCidrList(List<String> destinationCidrList) {
+        this.destinationCidrList = destinationCidrList;
+    }
+
+    public String getTrafficType() {
+        return trafficType;
+    }
+
+    public void setTrafficType(String trafficType) {
+        this.trafficType = trafficType;
+    }
+
+    public static class Builder {
+        public long domainId;
+        public long accountId;
+        public long zoneId;
+        public Long networkResourceId;
+        public String networkResourceName;
+        public boolean isVpcResource;
+        public long vmId;
+
+        public long ruleId;
+        public String publicIp;
+        public String vmIp;
+        public String publicPort;
+        public String privatePort;
+        public String protocol;
+        public String algorithm;
+        public List<String> sourceCidrList;
+        public List<String> destinationCidrList;
+        public String trafficType;
+        public Integer icmpType;
+        public Integer icmpCode;
+        public Network.Service service;
+
+        public Builder() {
+            // Default constructor
+        }
+
+        public Builder setDomainId(long domainId) {
+            this.domainId = domainId;
+            return this;
+        }
+
+        public Builder setAccountId(long accountId) {
+            this.accountId = accountId;
+            return this;
+        }
+
+        public Builder setZoneId(long zoneId) {
+            this.zoneId = zoneId;
+            return this;
+        }
+
+        public Builder setNetworkResourceId(Long networkResourceId) {
+            this.networkResourceId = networkResourceId;
+            return this;
+        }
+
+        public Builder setNetworkResourceName(String networkResourceName) {
+            this.networkResourceName = networkResourceName;
+            return this;
+        }
+
+        public Builder setVpcResource(boolean isVpcResource) {
+            this.isVpcResource = isVpcResource;
+            return this;
+        }
+
+
+        public Builder setVmId(long vmId) {
+            this.vmId = vmId;
+            return this;
+        }
+
+        public Builder setRuleId(long ruleId) {
+            this.ruleId = ruleId;
+            return this;
+        }
+
+        public Builder setPublicIp(String publicIp) {
+            this.publicIp = publicIp;
+            return this;
+        }
+
+        public Builder setVmIp(String vmIp) {
+            this.vmIp = vmIp;
+            return this;
+        }
+
+        public Builder setPublicPort(String publicPort) {
+            this.publicPort = publicPort;
+            return this;
+        }
+
+        public Builder setPrivatePort(String privatePort) {
+            this.privatePort = privatePort;
+            return this;
+        }
+
+        public Builder setProtocol(String protocol) {
+            this.protocol = protocol;
+            return this;
+        }
+
+        public Builder setAlgorithm(String algorithm) {
+            this.algorithm = algorithm;
+            return this;
+        }
+
+        public Builder setTrafficType(String trafficType) {
+            this.trafficType = trafficType;
+            return this;
+        }
+
+        public Builder setIcmpType(Integer icmpType) {
+            this.icmpType = icmpType;
+            return this;
+        }
+
+        public Builder setIcmpCode(Integer icmpCode) {
+            this.icmpCode = icmpCode;
+            return this;
+        }
+
+        public Builder setSourceCidrList(List<String> sourceCidrList) {
+            this.sourceCidrList = sourceCidrList;
+            return this;
+        }
+
+        public Builder setDestinationCidrList(List<String> destinationCidrList) {
+            this.destinationCidrList = destinationCidrList;
+            return this;
+        }
+
+        public Builder setService(Network.Service service) {
+            this.service = service;
+            return this;
+        }
+
+        public SDNProviderNetworkRule build() {
+            SDNProviderNetworkRule rule = new SDNProviderNetworkRule();
+            rule.setDomainId(this.domainId);
+            rule.setAccountId(this.accountId);
+            rule.setZoneId(this.zoneId);
+            rule.setNetworkResourceId(this.networkResourceId);
+            rule.setNetworkResourceName(this.networkResourceName);
+            rule.setVpcResource(this.isVpcResource);
+            rule.setVmId(this.vmId);
+            rule.setVmIp(this.vmIp);
+            rule.setPublicIp(this.publicIp);
+            rule.setPublicPort(this.publicPort);
+            rule.setPrivatePort(this.privatePort);
+            rule.setProtocol(this.protocol);
+            rule.setRuleId(this.ruleId);
+            rule.setAlgorithm(this.algorithm);
+            rule.setIcmpType(this.icmpType);
+            rule.setIcmpCode(this.icmpCode);
+            rule.setSourceCidrList(this.sourceCidrList);
+            rule.setDestinationCidrList(this.destinationCidrList);
+            rule.setTrafficType(this.trafficType);
+            rule.setService(service);
+            return rule;
+        }
+    }
+}

api/src/main/java/com/cloud/network/Site2SiteVpnConnection.java

@@ -24,7 +24,7 @@ import org.apache.cloudstack.api.InternalIdentity;
 
 public interface Site2SiteVpnConnection extends ControlledEntity, InternalIdentity, Displayable {
     enum State {
-        Pending, Connecting, Connected, Disconnected, Error,
+        Pending, Connecting, Connected, Disconnected, Error, Removed
     }
 
     @Override

api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java

@@ -87,4 +87,8 @@ public interface VirtualNetworkApplianceService {
     Pair<Boolean, String> performRouterHealthChecks(long routerId);
 
     <T extends VirtualRouter> void collectNetworkStatistics(T router, Nic nic);
+
+    enum RouterHealthStatus{
+        SUCCESS, FAILED, WARNING, UNKNOWN;
+    }
 }

api/src/main/java/com/cloud/network/as/AutoScaleService.java

@@ -70,6 +70,8 @@ public interface AutoScaleService {
 
     Counter createCounter(CreateCounterCmd cmd);
 
+    Counter getCounter(long counterId);
+
     boolean deleteCounter(long counterId) throws ResourceInUseException;
 
     List<? extends Counter> listCounters(ListCountersCmd cmd);

api/src/main/java/com/cloud/network/as/AutoScaleVmGroup.java

@@ -43,7 +43,7 @@ public interface AutoScaleVmGroup extends ControlledEntity, InternalIdentity, Di
             } else if (state.equalsIgnoreCase("scaling")) {
                 return SCALING;
             } else {
-                throw new IllegalArgumentException("Unexpected AutoScale VM group state : " + state);
+                throw new IllegalArgumentException("Unexpected AutoScale Instance group state : " + state);
             }
         }
     }

api/src/main/java/com/cloud/network/element/NetworkElement.java

@@ -23,6 +23,7 @@ import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
 import com.cloud.network.Network.Capability;
 import com.cloud.network.Network.Provider;
@@ -87,6 +88,14 @@ public interface NetworkElement extends Adapter {
     boolean release(Network network, NicProfile nic, VirtualMachineProfile vm, ReservationContext context) throws ConcurrentOperationException,
         ResourceUnavailableException;
 
+    /**
+     * Release IP from the network provider if reserved
+     * @param ipAddress
+     */
+    default boolean releaseIp(IpAddress ipAddress) {
+        return true;
+    }
+
     /**
      * The network is being shutdown.
      * @param network

api/src/main/java/com/cloud/network/element/PortForwardingServiceProvider.java

@@ -17,12 +17,40 @@
 package com.cloud.network.element;
 
 import java.util.List;
+import java.util.Objects;
 
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.Network;
+import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.PortForwardingRule;
+import com.cloud.network.vpc.NetworkACLItem;
 
 public interface PortForwardingServiceProvider extends NetworkElement, IpDeployingRequester {
+
+    static String getPublicPortRange(PortForwardingRule rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
+    static String getPrivatePFPortRange(PortForwardingRule rule) {
+        return rule.getDestinationPortStart() == rule.getDestinationPortEnd() ?
+                String.valueOf(rule.getDestinationPortStart()) :
+                String.valueOf(rule.getDestinationPortStart()).concat("-").concat(String.valueOf(rule.getDestinationPortEnd()));
+    }
+
+    static String getPrivatePortRange(FirewallRule rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
+    static String getPrivatePortRangeForACLRule(NetworkACLItem rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
     /**
      * Apply rules
      * @param network

api/src/main/java/com/cloud/network/element/VpcProvider.java

@@ -55,4 +55,8 @@ public interface VpcProvider extends NetworkElement {
     boolean applyACLItemsToPrivateGw(PrivateGateway gateway, List<? extends NetworkACLItem> rules) throws ResourceUnavailableException;
 
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+
+    default boolean updateVpc(Vpc vpc, String previousVpcName) {
+        return true;
+    }
 }

api/src/main/java/com/cloud/network/guru/NetworkGuru.java

@@ -215,4 +215,8 @@ public interface NetworkGuru extends Adapter {
     default boolean isSlaacV6Only() {
         return true;
     }
+
+    default boolean update(Network network, String prevNetworkName) {
+        return true;
+    }
 }

api/src/main/java/com/cloud/network/lb/LoadBalancingRulesService.java

@@ -41,13 +41,23 @@ import com.cloud.utils.net.Ip;
 public interface LoadBalancingRulesService {
     /**
      * Create a load balancer rule from the given ipAddress/port to the given private port
+     * @param xId an existing UUID for this rule (for instance a device generated one)
+     * @param name
+     * @param description
+     * @param srcPortStart
+     * @param srcPortEnd
+     * @param defPortStart
+     * @param defPortEnd
+     * @param ipAddrId
+     * @param protocol
+     * @param algorithm
+     * @param networkId
+     * @param lbOwnerId
      * @param openFirewall
-     *            TODO
+     * @param lbProtocol
-     * @param forDisplay TODO
+     * @param forDisplay
-     * @param cmd
-     *            the command specifying the ip address, public port, protocol, private port, and algorithm
-     *
      * @return the newly created LoadBalancerVO if successful, null otherwise
+     * @throws NetworkRuleConflictException
      * @throws InsufficientAddressCapacityException
      */
     LoadBalancer createPublicLoadBalancerRule(String xId, String name, String description, int srcPortStart, int srcPortEnd, int defPortStart, int defPortEnd,
@@ -106,7 +116,7 @@ public interface LoadBalancingRulesService {
 
     boolean applyLoadBalancerConfig(long lbRuleId) throws ResourceUnavailableException;
 
-    boolean assignCertToLoadBalancer(long lbRuleId, Long certId);
+    boolean assignCertToLoadBalancer(long lbRuleId, Long certId, boolean isForced);
 
     boolean removeCertFromLoadBalancer(long lbRuleId);
 

api/src/main/java/com/cloud/network/netris/NetrisLbBackend.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+public class NetrisLbBackend {
+    private long vmId;
+    private String vmIp;
+    private int port;
+
+    public NetrisLbBackend(long vmId, String vmIp, int port) {
+        this.vmId = vmId;
+        this.vmIp = vmIp;
+        this.port = port;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public String getVmIp() {
+        return vmIp;
+    }
+
+    public int getPort() {
+        return port;
+    }
+}

api/src/main/java/com/cloud/network/netris/NetrisNetworkRule.java

@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import com.cloud.network.SDNProviderNetworkRule;
+
+
+import java.util.List;
+
+public class NetrisNetworkRule {
+    public enum NetrisRuleAction {
+        PERMIT, DENY
+    }
+
+    private SDNProviderNetworkRule baseRule;
+    private NetrisRuleAction aclAction;
+    private List<NetrisLbBackend> lbBackends;
+    private String lbRuleName;
+    private String lbCidrList;
+    private String reason;
+
+    public NetrisNetworkRule(Builder builder) {
+        this.baseRule = builder.baseRule;
+        this.aclAction = builder.aclAction;
+        this.lbBackends = builder.lbBackends;
+        this.reason = builder.reason;
+        this.lbCidrList = builder.lbCidrList;
+        this.lbRuleName = builder.lbRuleName;
+    }
+
+    public NetrisRuleAction getAclAction() {
+        return aclAction;
+    }
+
+    public List<NetrisLbBackend> getLbBackends() {
+        return lbBackends;
+    }
+
+    public String getReason() {
+        return reason;
+    }
+
+    public String getLbCidrList() {return lbCidrList; }
+
+    public String getLbRuleName() { return lbRuleName; }
+
+    public SDNProviderNetworkRule getBaseRule() {
+        return baseRule;
+    }
+
+    // Builder class extending the parent builder
+    public static class Builder {
+        private SDNProviderNetworkRule baseRule;
+        private NetrisRuleAction aclAction;
+        private List<NetrisLbBackend> lbBackends;
+        private String reason;
+        private String lbCidrList;
+        private String lbRuleName;
+
+        public Builder baseRule(SDNProviderNetworkRule baseRule) {
+            this.baseRule = baseRule;
+            return this;
+        }
+
+        public Builder aclAction(NetrisRuleAction aclAction) {
+            this.aclAction = aclAction;
+            return this;
+        }
+
+        public Builder lbBackends(List<NetrisLbBackend> lbBackends) {
+            this.lbBackends = lbBackends;
+            return this;
+        }
+
+        public Builder reason(String reason) {
+            this.reason = reason;
+            return this;
+        }
+
+        public Builder lbCidrList(String lbCidrList) {
+            this.lbCidrList = lbCidrList;
+            return this;
+        }
+
+        public Builder lbRuleName(String lbRuleName) {
+            this.lbRuleName = lbRuleName;
+            return this;
+        }
+
+        public NetrisNetworkRule build() {
+            return new NetrisNetworkRule(this);
+        }
+    }
+}

api/src/main/java/com/cloud/network/netris/NetrisProvider.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface NetrisProvider extends InternalIdentity, Identity {
+    long getZoneId();
+    String getName();
+    String getUrl();
+    String getUsername();
+    String getSiteName();
+    String getTenantName();
+    String getNetrisTag();
+}

api/src/main/java/com/cloud/network/netris/NetrisService.java

@@ -0,0 +1,310 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import com.cloud.network.IpAddress;
+import com.cloud.network.Network;
+import com.cloud.network.SDNProviderNetworkRule;
+import com.cloud.network.vpc.StaticRoute;
+import com.cloud.network.vpc.Vpc;
+
+import java.util.List;
+
+/**
+ * Interface for Netris Services that provides methods to manage VPCs, networks,
+ * NAT rules, network rules, and static routes in an SDN (Software Defined Networking) environment.
+ */
+
+public interface NetrisService {
+
+    /**
+     * Creates IPAM (IP Address Management) allocations for zone-level public ranges.
+     *
+     * @param zoneId the ID of the zone
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createIPAMAllocationsForZoneLevelPublicRanges(long zoneId);
+
+    /**
+     * Creates a VPC (Virtual Private Cloud) resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcId            the ID of the VPC
+     * @param vpcName          the name of the VPC
+     * @param sourceNatEnabled true if source NAT is enabled
+     * @param cidr             the CIDR of the VPC
+     * @param isVpcNetwork     true if it is a VPC network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createVpcResource(long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled, String cidr, boolean isVpcNetwork);
+
+    /**
+     * Updates an existing VPC resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcId            the ID of the VPC
+     * @param vpcName          the new name of the VPC
+     * @param previousVpcName  the previous name of the VPC
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVpcResource(long zoneId, long accountId, long domainId, Long vpcId, String vpcName, String previousVpcName);
+
+    /**
+     * Deletes a VPC resource.
+     *
+     * @param zoneId   the ID of the zone
+     * @param accountId the ID of the account
+     * @param domainId the ID of the domain
+     * @param vpc      the VPC to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteVpcResource(long zoneId, long accountId, long domainId, Vpc vpc);
+
+    /**
+     * Creates a virtual network (vNet) resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcName          the name of the VPC
+     * @param vpcId            the ID of the VPC
+     * @param networkName      the name of the network
+     * @param networkId        the ID of the network
+     * @param cidr             the CIDR of the network
+     * @param globalRouting    true if global routing is enabled
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr, Boolean globalRouting);
+
+    /**
+     * Updates an existing vNet resource.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the new name of the network
+     * @param networkId       the ID of the network
+     * @param prevNetworkName the previous name of the network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String prevNetworkName);
+
+    /**
+     * Deletes an existing vNet resource.
+     *
+     * @param zoneId       the ID of the zone
+     * @param accountId    the ID of the account
+     * @param domainId     the ID of the domain
+     * @param vpcName      the name of the VPC
+     * @param vpcId        the ID of the VPC
+     * @param networkName  the name of the network
+     * @param networkId    the ID of the network
+     * @param cidr         the CIDR of the network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteVnetResource(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr);
+
+    /**
+     * Creates a source NAT rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param sourceNatIp     the source NAT IP
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createSnatRule(long zoneId, long accountId, long domainId, String vpcName, long vpcId, String networkName, long networkId, boolean isForVpc, String vpcCidr, String sourceNatIp);
+
+    /**
+     * Creates a port forwarding rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param networkRule     the network rule to forward
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createPortForwardingRule(long zoneId, long accountId, long domainId, String vpcName, long vpcId, String networkName, Long networkId, boolean isForVpc, String vpcCidr, SDNProviderNetworkRule networkRule);
+
+    /**
+     * Deletes a port forwarding rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param networkRule     the network rule to remove
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deletePortForwardingRule(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, boolean isForVpc, String vpcCidr, SDNProviderNetworkRule networkRule);
+
+    /**
+     * Updates the source NAT IP for a specified VPC.
+     *
+     * @param vpc     the VPC to updates
+     * @param address the new source NAT IP address
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+
+    /**
+     * Creates a static NAT rule for a specific VM.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if the rule applies to a VPC
+     * @param vpcCidr            the VPC CIDR
+     * @param staticNatIp        the static NAT IP
+     * @param vmIp               the VM's IP address
+     * @param vmId               the ID of the VM
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createStaticNatRule(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String vpcCidr, String staticNatIp, String vmIp, long vmId);
+
+    /**
+     * Deletes a static NAT rule for a specific VM.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if the rule applies to a VPC
+     * @param staticNatIp        the static NAT IP
+     * @param vmId               the ID of the VM
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteStaticNatRule(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String staticNatIp, long vmId);
+
+    /**
+     * Adds firewall rules to a specific network.
+     *
+     * @param network       the target network
+     * @param firewallRules the list of firewall rules to add
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean addFirewallRules(Network network, List<NetrisNetworkRule> firewallRules);
+
+    /**
+     * Deletes firewall rules from a specific network.
+     *
+     * @param network       the target network
+     * @param firewallRules the list of firewall rules to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteFirewallRules(Network network, List<NetrisNetworkRule> firewallRules);
+
+    /**
+     * Adds or updates a static route for a specific network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @param updateRoute        true if the route should be updated
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean addOrUpdateStaticRoute(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId, boolean updateRoute);
+
+    /**
+     * Deletes a specific static route for a network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteStaticRoute(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId);
+
+    /**
+     * Lists static routes for a specific network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @return a list of static routes
+     */
+    List<StaticRoute> listStaticRoutes(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId);
+
+    /**
+     * Releases a NAT IP address.
+     *
+     * @param zoneId   the ID of the zone
+     * @param publicIp the public NAT IP to release
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean releaseNatIp(long zoneId, String publicIp);
+
+    /**
+     * Creates or updates a load balancer (LB) rule.
+     *
+     * @param rule the network rule for the load balancer
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createOrUpdateLbRule(NetrisNetworkRule rule);
+
+    /**
+     * Deletes a load balancer (LB) rule.
+     *
+     * @param rule the network rule to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteLbRule(NetrisNetworkRule rule);
+}

api/src/main/java/com/cloud/network/nsx/NsxService.java

@@ -16,9 +16,10 @@
 // under the License.
 package com.cloud.network.nsx;
 
+import org.apache.cloudstack.framework.config.ConfigKey;
+
 import com.cloud.network.IpAddress;
 import com.cloud.network.vpc.Vpc;
-import org.apache.cloudstack.framework.config.ConfigKey;
 
 public interface NsxService {
 
@@ -33,4 +34,5 @@ public interface NsxService {
 
     boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled);
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+    String getSegmentId(long domainId, long accountId, long zoneId, Long vpcId, long networkId);
 }

api/src/main/java/com/cloud/network/rules/LbStickinessMethod.java

@@ -108,8 +108,7 @@ public class LbStickinessMethod {
     }
 
     public void addParam(String name, Boolean required, String description, Boolean isFlag) {
-        /* FIXME : UI is breaking if the capability string length is larger , temporarily description is commented out */
+        /* is this still a valid comment: FIXME : UI is breaking if the capability string length is larger , temporarily description is commented out */
-        // LbStickinessMethodParam param = new LbStickinessMethodParam(name, required, description);
         LbStickinessMethodParam param = new LbStickinessMethodParam(name, required, " ", isFlag);
         _paramList.add(param);
         return;
@@ -133,7 +132,6 @@ public class LbStickinessMethod {
 
     public void setDescription(String description) {
         /* FIXME : UI is breaking if the capability string length is larger , temporarily description is commented out */
-        //this.description = description;
         this._description = " ";
     }
 }

api/src/main/java/com/cloud/network/vpc/StaticRoute.java

@@ -25,6 +25,7 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit
         Staged, // route been created but has never got through network rule conflict detection.  Routes in this state can not be sent to VPC virtual router.
         Add,    // Add means the route has been created and has gone through network rule conflict detection.
         Active, // Route has been sent to the VPC router and reported to be active.
+        Update,
         Revoke,  // Revoke means this route has been revoked. If this route has been sent to the VPC router, the route will be deleted from database.
         Deleting // rule has been revoked and is scheduled for deletion
     }
@@ -32,7 +33,9 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit
     /**
      * @return
      */
-    long getVpcGatewayId();
+    Long getVpcGatewayId();
+
+    String getNextHop();
 
     /**
      * @return

api/src/main/java/com/cloud/network/vpc/StaticRouteProfile.java

@@ -23,7 +23,8 @@ public class StaticRouteProfile implements StaticRoute {
     private String targetCidr;
     private long accountId;
     private long domainId;
-    private long gatewayId;
+    private Long gatewayId;
+    private String nextHop;
     private StaticRoute.State state;
     private long vpcId;
     String vlanTag;
@@ -46,6 +47,18 @@ public class StaticRouteProfile implements StaticRoute {
         ipAddress = gateway.getIp4Address();
     }
 
+    public StaticRouteProfile(StaticRoute staticRoute) {
+        id = staticRoute.getId();
+        uuid = staticRoute.getUuid();
+        targetCidr = staticRoute.getCidr();
+        accountId = staticRoute.getAccountId();
+        domainId = staticRoute.getDomainId();
+        gatewayId = staticRoute.getVpcGatewayId();
+        state = staticRoute.getState();
+        vpcId = staticRoute.getVpcId();
+        gateway = staticRoute.getNextHop();
+    }
+
     @Override
     public long getAccountId() {
         return accountId;
@@ -57,10 +70,15 @@ public class StaticRouteProfile implements StaticRoute {
     }
 
     @Override
-    public long getVpcGatewayId() {
+    public Long getVpcGatewayId() {
         return gatewayId;
     }
 
+    @Override
+    public String getNextHop() {
+        return nextHop;
+    }
+
     @Override
     public String getCidr() {
         return targetCidr;