Merge branch 'main' into keyboard-shortcuts

.asf.yaml

@@ -51,13 +51,14 @@ github:
 
   collaborators:
     - acs-robot
-    - kiranchavala
-    - rajujith
-    - GaOrtiga
-    - SadiJr
-    - winterhazel
     - gpordeus
     - hsato03
+    - FelipeM525
+    - lucas-a-martins
+    - nicoschmdt
+    - abh1sar
+    - rosi-shapeblue
+    - sudo87
 
   protected_branches: ~
 

.github/ISSUE_TEMPLATE/bug.yml

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+description: "Thank you for reporting a bug!"
+name: bug
+title: "[SHORT PROBLEM DESCRIPTION]"
+labels: bug, needs-triageing
+body:
+- type: markdown
+  attributes:
+    value: "## Welcome, please describe your problem below;"
+- type: textarea
+  attributes:
+    label: problem
+    value: The long description of your problem
+- type: markdown
+  attributes:
+    value: "## What versions of cloudstack and any infra components are you using"
+- type: textarea
+  attributes:
+    label: versions
+    value: The versions of ACS, hypervisors, storage, network etc..
+- type: textarea
+  attributes:
+    label: The steps to reproduce the bug
+    value: |
+      1.
+      2.
+      3.
+      ...
+- type: textarea
+  attributes:
+    label: "What to do about it?"

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+description: "Thank you for your new feature idea!"
+name: feature
+title: "[SHORT FUNCTIONAL DESCRIPTION]"
+labels:  new
+body:
+- type: textarea
+  attributes:
+    label: "The required feature described as a wish"
+    value: As a User/Admin/Operator I would like to , ... have the system make my morning coffee.

.github/linters/.flake8

@@ -22,8 +22,11 @@
 # E224 Tab after operator
 # E227 Missing whitespace around bitwise or shift operator
 # E242 Tab after ','
+# E271 Multiple spaces after keyword
+# E272 Multiple spaces before keyword
 # E273 Tab after keyword
 # E274 Tab before keyword
+# E713 Test for membership should be 'not in'
 # E742 Do not define classes named 'I', 'O', or 'l'
 # E743 Do not define functions named 'I', 'O', or 'l'
 # E901 SyntaxError or IndentationError
@@ -37,4 +40,4 @@
 exclude =
     .git,
     venv
-select = E112,E113,E133,E223,E224,E227,E242,E273,E274,E742,E743,E901,E902,W291,W292,W293,W391
+select = E112,E113,E133,E223,E224,E227,E242,E271,E272,E273,E274,E713,E742,E743,E901,E902,W291,W292,W293,W391

.github/linters/.markdown-lint.yml

@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# MD001/heading-increment Heading levels should only increment by one level at a time
+MD001: false
+
+# MD003/heading-style Heading style
+MD003: false
+
+# MD004/ul-style Unordered list style
+MD004: false
+
+# MD007/ul-indent Unordered list indentation
+MD007: false
+
+# MD010/no-hard-tabs Hard tabs
+MD010: false
+
+# MD013/line-length Line length
+MD013: false
+
+# MD014/commands-show-output Dollar signs used before commands without showing output
+MD014: false
+
+# MD022/blanks-around-headings Headings should be surrounded by blank lines
+MD022: false
+
+# MD023/heading-start-left Headings must start at the beginning of the line
+MD023: false
+
+# MD024/no-duplicate-heading Multiple headings with the same content
+MD024: false
+
+# MD025/single-title/single-h1 Multiple top-level headings in the same document
+MD025: false
+
+# MD026/no-trailing-punctuation Trailing punctuation in heading
+MD026: false
+
+# MD028/no-blanks-blockquote Blank line inside blockquote
+MD028: false
+
+# MD029/ol-prefix Ordered list item prefix
+MD029: false
+
+# MD031/blanks-around-fences Fenced code blocks should be surrounded by blank lines
+MD031: false
+
+# MD032/blanks-around-lists Lists should be surrounded by blank lines
+MD032: false
+
+# MD033/no-inline-html Inline HTML
+MD033: false
+
+# MD034/no-bare-urls Bare URL used
+MD034: false
+
+# MD036/no-emphasis-as-heading Emphasis used instead of a heading
+MD036: false
+
+# MD037/no-space-in-emphasis Spaces inside emphasis markers
+MD037: false
+
+# MD040/fenced-code-language Fenced code blocks should have a language specified
+MD040: false
+
+# MD041/first-line-heading/first-line-h1 First line in a file should be a top-level heading
+MD041: false
+
+# MD046/code-block-style Code block style
+MD046: false
+
+# MD052/reference-links-images Reference links and images should use a label that is defined
+MD052: false
+
+# MD059/descriptive-link-text Link text should be descriptive
+MD059: false

.github/linters/codespell.txt

@@ -0,0 +1,526 @@
+accouns
+acheived
+acount
+actuall
+acuiring
+acumulate
+addreess
+addtion
+adminstrator
+afer
+afrer
+afterall
+againt
+ags
+aktive
+algoritm
+allo
+alloacate
+allocted
+alocation
+alogrithm
+alpha-numeric
+alue
+ammended
+ammount
+ans
+anull
+apche
+aplication
+apllication
+applicatio
+apporpriate
+appropritate
+aqcuire
+aqcuired
+aquire
+aquiring
+assiciate
+assigne
+assoication
+assosiate
+asssert
+astroid
+asynchroniously
+asyncronous
+atleast
+atomation
+attache
+attch
+attches
+authenciation
+authenitcation
+authenitication
+availiability
+avialable
+bais
+beacause
+beacuse
+becase
+becasue
+becaues
+behviour
+birdge
+bject
+boardcast
+bootstraper
+bu
+cant
+capabilites
+capablity
+capcity
+carrefully
+cavaet
+chaing
+checkd
+childs
+choosen
+chould
+clenup
+cliente
+clinet
+cluser
+cna
+collison
+comman
+commited
+comparision
+comparisions
+complient
+concious
+conectix
+confg
+configruation
+configuable
+conneciton
+connexion
+constrait
+constraits
+containg
+contex
+continuesly
+contro
+controler
+controles
+controll
+convienient
+convinience
+coputer
+correcponding
+correspoding
+correspoonds
+cosole
+coudl
+couldnt
+craete
+craeted
+crate
+crated
+createa
+createing
+credentail
+cros
+crresponding
+curren
+currentl
+datas
+decalared
+declatory
+decocdes
+decypher
+defalut
+defaut
+defered
+definiton
+deleteable
+dependancy
+dependant
+dependend
+deployement
+deply
+deplying
+dervied
+descktop
+descrption
+deserialzed
+desination
+detination
+detroy
+detroying
+dettach
+dettached
+dettaching
+diabling
+diasbled
+dictonary
+didnt
+differnet
+differnt
+direcotry
+directroy
+disale
+disbale
+discrepency
+disover
+dissapper
+dissassociated
+divice
+doesn'
+doesnot
+doesnt
+dont'
+doubleclick
+dows
+eanbled
+earch
+ect
+elemnt
+eles
+elments
+emmited
+enble
+encryted
+enebled
+enmpty
+entires
+enviornment
+environmnet
+equivalant
+erro
+erronous
+everthing
+everytime
+excetion
+excption
+excute
+execept
+execption
+execut
+executeable
+exeeded
+exisitng
+exisits
+existin
+existsing
+exitting
+expcted
+expection
+explaination
+explicitely
+faield
+faild
+failes
+falied
+fasion
+feild
+filenname
+fillled
+findout
+fisrt
+fo
+folowing
+fowarding
+frist
+fro
+frontent
+fuctionality
+genarate
+generallly
+gernerate
+get's
+gloabal
+gorry
+gracefull
+gradiant
+handeling
+hanling
+happend
+hasing
+hasnt
+hda
+hostanme
+hould
+hsould
+hte
+identifers
+identifyer
+identifyers
+igoring
+immediatley
+implememented
+implementor
+implementors
+implemnt
+implict
+implmeneted
+implmentation
+incase
+includeing
+incosistency
+indecates
+indien
+infor
+informations
+informaton
+infrastrcuture
+ingore
+inital
+initalize
+initator
+initilization
+inspite
+instace
+instal
+instnace
+intefaces
+intepret
+intereface
+interfer
+interpretted
+intialize
+intializes
+intializing
+invlaid
+invokation
+isnt
+ist
+klunky
+lable
+leve
+lief
+limite
+linke
+listner
+lokal
+lokales
+maintainence
+maintenace
+maintenence
+mamagement
+mambers
+manaully
+manuel
+maxium
+mehtod
+mergable
+mesage
+messge
+metatdata
+milisecond
+minumum
+mis
+modifers
+mor
+mot
+mulitply
+multipl
+multple
+mutliple
+nast
+nd
+neccessary
+necesary
+netowrk
+nin
+nodel
+nome
+noone
+nowe
+numbe
+numer
+occured
+occurence
+occuring
+offfering
+ofthe
+omited
+onother
+opeation
+optin
+orginal
+otherwse
+outter
+overriden
+overwritting
+paramater
+paramemeter
+paramenter
+paramete
+parametrs
+pararmeter
+parms
+parralels
+particualr
+passowrd
+perfromed
+permissble
+physcial
+plugable
+pluging
+polcies
+policys
+poluting
+possiblity
+potenial
+prefered
+preffered
+pressenter
+previleges
+primay
+priviledged
+procuct
+programatically
+progres
+properites
+propertie
+propertys
+propogate
+provison
+psudo
+pyhsical
+readabilty
+readd
+reccuring
+recevied
+recieved
+recursivelly
+redunant
+refference
+releease
+relese
+remaning
+remore
+remvoing
+renabling
+repeatly
+reponse
+reqest
+reqiured
+requieres
+requried
+reserv
+reserverd
+reseted
+reseting
+resorce
+responser
+resposne
+resturns
+retreive
+retreiving
+retrive
+retrived
+retriving
+retrun
+retuned
+returing
+re-use
+rever
+rocessor
+runing
+runnign
+sate
+scalled
+scipt
+scirpt
+scrip
+seconadry
+seconday
+seesion
+sepcified
+sepcify
+seprated
+ser
+servies
+seting
+settig
+sevices
+shoul
+shoule
+sie
+signle
+simplier
+singature
+skiping
+snaphsot
+snpashot
+specied
+specifed
+specifiy
+splitted
+spped
+standy
+statics
+stickyness
+stil
+stip
+storeage
+strat
+streched
+strutural
+succesfull
+successfull
+suceessful
+suces
+sucessfully
+suiteable
+suppots
+suppport
+syncronous
+syste
+tage
+te
+tempdate
+testng
+tha
+thats
+ther
+therefor
+theres
+theses
+thi
+thorugh
+throught
+ths
+tipically
+transction
+tring
+trough
+tyoe
+ue
+ues
+unavailibility
+uncommited
+uncompressible
+uneccessarily
+unexepected
+unexpect
+unknow
+unkonw
+unkown
+unneccessary
+unparseable
+unrecoginized
+unsupport
+unxpected
+updat
+uptodate
+usera
+usign
+usin
+utlization
+vaidate
+valiate
+valule
+valus
+varibles
+verfy
+verfying
+verifing
+virutal
+visable
+wakup
+wil
+wit
+wll
+wth

.github/workflows/build.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
           architecture: 'x64'
 
       - name: Install Build Dependencies

.github/workflows/ci.yml

@@ -35,6 +35,7 @@ jobs:
       fail-fast: false
       matrix:
         tests: [ "smoke/test_accounts
+                  smoke/test_account_access
                   smoke/test_affinity_groups
                   smoke/test_affinity_groups_projects
                   smoke/test_annotations
@@ -54,6 +55,7 @@ jobs:
                   smoke/test_deploy_vm_with_userdata
                   smoke/test_deploy_vms_in_parallel
                   smoke/test_deploy_vms_with_varied_deploymentplanners
+                  smoke/test_restore_vm
                   smoke/test_diagnostics
                   smoke/test_direct_download
                   smoke/test_disk_offerings
@@ -85,7 +87,12 @@ jobs:
                   smoke/test_migration
                   smoke/test_multipleips_per_nic
                   smoke/test_nested_virtualization
-                  smoke/test_set_sourcenat",
+                  smoke/test_set_sourcenat
+                  smoke/test_webhook_lifecycle
+                  smoke/test_purge_expunged_vms
+                  smoke/test_extension_lifecycle
+                  smoke/test_extension_custom_action_lifecycle
+                  smoke/test_extension_custom",
                 "smoke/test_network
                   smoke/test_network_acl
                   smoke/test_network_ipv6
@@ -131,7 +138,9 @@ jobs:
                   smoke/test_usage
                   smoke/test_usage_events
                   smoke/test_vm_deployment_planner
+                  smoke/test_vm_strict_host_tags
                   smoke/test_vm_schedule
+                  smoke/test_deploy_vgpu_enabled_vm
                   smoke/test_vm_life_cycle
                   smoke/test_vm_lifecycle_unmanage_import
                   smoke/test_vm_snapshot_kvm
@@ -159,7 +168,8 @@ jobs:
                   component/test_cpu_limits
                   component/test_cpu_max_limits
                   component/test_cpu_project_limits
-                  component/test_deploy_vm_userdata_multi_nic",
+                  component/test_deploy_vm_userdata_multi_nic
+                  component/test_deploy_vm_lease",
                 "component/test_egress_fw_rules
                   component/test_invalid_gw_nm
                   component/test_ip_reservation",
@@ -221,7 +231,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
           architecture: 'x64'
 
       - name: Install Build Dependencies
@@ -231,7 +241,7 @@ jobs:
 
       - name: Install Python dependencies
         run: |
-          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycrypto mock flask netaddr pylint pycodestyle six astroid
+          python3 -m pip install --user --upgrade urllib3 lxml paramiko nose texttable ipmisim pyopenssl pycryptodome mock flask netaddr pylint pycodestyle six astroid pynose
 
       - name: Install jacoco dependencies
         run: |
@@ -279,7 +289,7 @@ jobs:
 
       - name: Start CloudStack Management Server with Simulator
         run: |
-          export MAVEN_OPTS="-Xmx4096m -XX:MaxPermSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver"
+          export MAVEN_OPTS="-Xmx4096m -XX:MaxMetaspaceSize=800m -Djava.security.egd=file:/dev/urandom  -javaagent:jacoco/lib/jacocoagent.jar=address=*,port=36320,output=tcpserver --add-opens=java.base/java.lang=ALL-UNNAMED --add-exports=java.base/sun.security.x509=ALL-UNNAMED --add-opens=java.base/jdk.internal.reflect=ALL-UNNAMED"
           echo -e "\nStarting simulator"
           set +e
           mvn -Dsimulator -Dorg.eclipse.jetty.annotations.maxWait=120 -pl :cloud-client-ui jetty:run 2>&1 > /tmp/jetty-log || true &
@@ -313,10 +323,11 @@ jobs:
           echo -e "Simulator CI Test Results: (only failures listed)\n"
           python3 ./tools/marvin/xunit-reader.py integration-test-results/
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           files: jacoco-coverage.xml
           fail_ci_if_error: true
           flags: simulator-marvin-tests
           verbose: true
           name: codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/codecov.yml

@@ -36,11 +36,11 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up JDK11
+      - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: '17'
           cache: 'maven'
 
       - name: Build CloudStack with Quality Checks
@@ -49,10 +49,11 @@ jobs:
           cd nonoss && bash -x install-non-oss.sh && cd ..
           mvn -P quality -Dsimulator -Dnoredist clean install -T$(nproc)
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
         with:
           files: ./client/target/site/jacoco-aggregate/jacoco.xml
           fail_ci_if_error: true
-          flags: unit-tests
+          flags: unittests
           verbose: true
           name: codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/linter.yml

@@ -39,7 +39,7 @@ jobs:
           pip install pre-commit
       - name: Set PY
         run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}

.github/workflows/main-sonar-check.yml

@@ -36,25 +36,25 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up JDK11
+      - name: Set up JDK17
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: '17'
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
       - name: Cache local Maven repository
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

.github/workflows/rat.yml

@@ -31,10 +31,10 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - name: Set up JDK 11
+      - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
-          java-version: '11'
+          java-version: '17'
           distribution: 'adopt'
           architecture: x64
           cache: maven

.github/workflows/sonar-check.yml

@@ -38,25 +38,25 @@ jobs:
           ref: "refs/pull/${{ github.event.number }}/merge"
           fetch-depth: 0
 
-      - name: Set up JDK11
+      - name: Set up JDK17
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
-          java-version: '11'
+          java-version: '17'
           cache: 'maven'
 
       - name: Cache SonarCloud packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.sonar/cache
           key: ${{ runner.os }}-sonar
           restore-keys: ${{ runner.os }}-sonar
 
       - name: Cache local Maven repository
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
-          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          key: ${{ runner.os }}-m2-${{ hashFiles('pom.xml', '*/pom.xml', '*/*/pom.xml', '*/*/*/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2
 

.github/workflows/ui.yml

@@ -36,7 +36,7 @@ jobs:
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
-          node-version: 14
+          node-version: 16
 
       - name: Env details
         run: |
@@ -55,7 +55,8 @@ jobs:
           npm run lint
           npm run test:unit
 
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
+        if: github.repository == 'apache/cloudstack'
         with:
           working-directory: ui
           files: ./coverage/lcov.info
@@ -63,3 +64,4 @@ jobs:
           flags: uitests
           verbose: true
           name: codecov
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -15,92 +15,92 @@
 # specific language governing permissions and limitations
 # under the License.
 
-build/build.number
-.lock-wscript
-.waf-*
-waf-*
-target/
-override/
-.metadata
-dist/
 *~
+*_flymake.js
 *.bak
-cloud-*.tar.bz2
-*.log
-*.pyc
-*.patch
+*.css.map
 *.egginfo/
 *.egg-info/
-*.prefs
-build.number
-*.log.*
-cloud.log.*.*
-unittest
-deps/cloud.userlibraries
-.DS_Store
-.idea
 *.iml
-git-remote-https.exe.stackdump
-*.swp
-tools/devcloud/devcloudbox/.vagrant
-tools/cli/cloudmonkey/marvin/
-tools/cli/cloudmonkey/precache.py
-tools/marvin/marvin/cloudstackAPI/
-tools/marvin/build/
-tools/cli/build/
-tools/appliance/systemvmtemplate/packer_cache/
-*.jar
-*.war
-*.mar
 *.iso
+*.jar
+*.log
+*.log.*
+*.mar
+*.orig
+*.patch
+*.prefs
+*.pyc
+*.qcow2
+*.raw
+*.swp
 *.tar.gz
 *.tgz
 *.vscode
-*.css.map
+*.war
+.DS_Store
+.checkstyle
+.classpath
+.idea
+.lock-wscript
+.metadata
+.pmd
+.pmdruleset.xml
+.project
+.pydevproject
+.reviewboardrc
+.settings.xml
+.settings/
+.vscode
+.waf-*
+Gemfile.lock
+build/build.number
+build.number
+build-indep-stamp
+cloud.log.*.*
+cloud-*.tar.bz2
+configure-stamp
+db.properties.override
+debian/*.debhelper
+debian/*.substvars
+debian/cloudstack-*/*
+debian/files
+debian/tmp
+deps/cloud.userlibraries
+dist/
+docs/publish
+docs/runbook/publish
+docs/runbook/tmp
+docs/tmp
+engine/storage/integration-test/test-output
+git-remote-https.exe.stackdump
+node_modules
+override/
+plugins/hypervisors/kvm/.pydevproject
+plugins/network-elements/juniper-contrail/logs/
+replace.properties.override
+replace.properties.tmp
+scripts/.pydevproject
+scripts/vm/hypervisor/xenserver/vhd-util
+systemvm/.pydevproject
+target/
+target-eclipse
+test/.pydevprojec
+tools/apidoc/log/
+tools/appliance/box/
+tools/appliance/systemvmtemplate/packer_cache/
+tools/cli/build/
+tools/cli/cloudmonkey/marvin/
+tools/cli/cloudmonkey/precache.py
+tools/devcloud/devcloudbox/.vagrant
+tools/marvin/build/
+tools/marvin/marvin/cloudstackAPI/
+tools/marvin/marvin/cloudstackAPI/*
+unittest
+venv
+waf-*
 
 # this ignores _all files starting with '.'. Don't do that!
 #.*
 
-target-eclipse
 !.gitignore
-.classpath
-.settings.xml
-.settings/
-db.properties.override
-replace.properties.override
-tools/marvin/marvin/cloudstackAPI/*
-docs/tmp
-docs/publish
-docs/runbook/tmp
-docs/runbook/publish
-.project
-Gemfile.lock
-debian/tmp
-debian/files
-debian/cloudstack-*/*
-debian/*.substvars
-debian/*.debhelper
-replace.properties.tmp
-build-indep-stamp
-configure-stamp
-*_flymake.js
-engine/storage/integration-test/test-output
-tools/apidoc/log/
-plugins/network-elements/juniper-contrail/logs/
-scripts/vm/hypervisor/xenserver/vhd-util
-*.orig
-tools/appliance/box/
-.reviewboardrc
-.checkstyle
-.pmd
-.pmdruleset.xml
-.pydevproject
-systemvm/.pydevproject
-test/.pydevprojec
-plugins/hypervisors/kvm/.pydevproject
-scripts/.pydevproject
-*.qcow2
-*.raw
-venv
-node_modules
-.vscode

.pre-commit-config.yaml

@@ -15,18 +15,24 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   # force all unspecified Python hooks to run python3
   python: python3
-minimum_pre_commit_version: "2.17.0"
+minimum_pre_commit_version: "3.2.0"
 repos:
   - repo: meta
     hooks:
       - id: identity
       - id: check-hooks-apply
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.27.2
+    hooks:
+      - id: gitleaks
+        name: run gitleaks
+        description: detect hardcoded secrets
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v5.0.0
     hooks:
       #- id: check-added-large-files
       - id: check-case-conflict
@@ -36,6 +42,8 @@ repos:
       - id: check-vcs-permalinks
       #- id: check-yaml
       - id: destroyed-symlinks
+      - id: detect-aws-credentials
+        args: [--allow-missing-credentials]
       - id: detect-private-key
         exclude: >
           (?x)
@@ -52,12 +60,24 @@ repos:
           ^systemvm/agent/certs/realhostip\.key$
       - id: end-of-file-fixer
         exclude: \.vhd$
-      #- id: fix-byte-order-marker
+      - id: fix-byte-order-marker
+      - id: forbid-submodules
       - id: mixed-line-ending
-        exclude: \.(cs|xml)$
-      # - id: trailing-whitespace
+        exclude: \.cs$
+      - id: trailing-whitespace
+        files: \.(bat|cfg|cs|css|gitignore|header|in|install|java|md|properties|py|rb|sh|sql|txt|vue|xml|xsl|yaml|yml)$
+        args: [--markdown-linebreak-ext=md]
+        exclude: ^services/console-proxy/rdpconsole/src/test/doc/freerdp-debug-log\.txt$
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        name: run codespell
+        description: Check spelling with codespell
+        args: [--ignore-words=.github/linters/codespell.txt]
+        exclude: ^systemvm/agent/noVNC/|^ui/package\.json$|^ui/package-lock\.json$|^ui/public/js/less\.min\.js$|^ui/public/locales/.*[^n].*\.json$
   - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
+    rev: 7.0.0
     hooks:
     - id: flake8
       args: [--config, .github/linters/.flake8]
@@ -72,3 +92,12 @@ repos:
         ^scripts/vm/hypervisor/xenserver/vmopspremium$|
         ^setup/bindir/cloud-setup-encryption\.in$|
         ^venv/.*$
+  - repo: https://github.com/igorshubovych/markdownlint-cli
+    rev: v0.45.0
+    hooks:
+      - id: markdownlint
+        name: run markdownlint
+        description: check Markdown files with markdownlint
+        args: [--config=.github/linters/.markdown-lint.yml]
+        types: [markdown]
+        files: \.(md|mdown|markdown)$

.python-version

@@ -1 +1 @@
-3.6
+3.10

CHANGES.md

@@ -242,7 +242,6 @@ Bug ID | Description
 [CLOUDSTACK-7722](https://issues.apache.org/jira/browse/CLOUDSTACK-7722) | add.label: Add button for tags show the label not "Add" text...
 [CLOUDSTACK-7246](https://issues.apache.org/jira/browse/CLOUDSTACK-7246) | VM deployment failed due to wrong in  script name createipalias.sh...
 
-
 Version 4.4.1
 -------------
 
@@ -276,7 +275,6 @@ Bug ID | Description
 [CLOUDSTACK-1632](https://issues.apache.org/jira/browse/CLOUDSTACK-1632) | Mistakes in authorizeSecurityGroup* API docs...
 [CLOUDSTACK-401](https://issues.apache.org/jira/browse/CLOUDSTACK-401)   | Storage options missing from table...
 
-
 Version 4.4.0
 -------------
 
@@ -430,11 +428,11 @@ Bug ID | Description
 [CLOUDSTACK-6099](https://issues.apache.org/jira/browse/CLOUDSTACK-6099) | live migration is failing for vm deployed using dynamic compute offerings with NPE
 [CLOUDSTACK-7528](https://issues.apache.org/jira/browse/CLOUDSTACK-7528) | More verbose logging when sending alert fails
 [CLOUDSTACK-6624](https://issues.apache.org/jira/browse/CLOUDSTACK-6624) | set specifyIpRanges to true if specifyVlan is set to true
-[CLOUDSTACK-7404](https://issues.apache.org/jira/browse/CLOUDSTACK-7404) | Failed to start an instance when originating template has been deleted 
+[CLOUDSTACK-7404](https://issues.apache.org/jira/browse/CLOUDSTACK-7404) | Failed to start an instance when originating template has been deleted
 [CLOUDSTACK-6531](https://issues.apache.org/jira/browse/CLOUDSTACK-6531) | Stopping the router in case of command failures
 [CLOUDSTACK-6115](https://issues.apache.org/jira/browse/CLOUDSTACK-6115) | TravisCI configuration
 [CLOUDSTACK-7405](https://issues.apache.org/jira/browse/CLOUDSTACK-7405) | allowing VR meta-data to be accessed without trailing slash
-[CLOUDSTACK-7260](https://issues.apache.org/jira/browse/CLOUDSTACK-7260) | Management server not responding after some time for Vmware due to Oom 
+[CLOUDSTACK-7260](https://issues.apache.org/jira/browse/CLOUDSTACK-7260) | Management server not responding after some time for Vmware due to Oom
 [CLOUDSTACK-7038](https://issues.apache.org/jira/browse/CLOUDSTACK-7038) | Add mysql client dependency for mgmt server pkg for debian
 [CLOUDSTACK-6892](https://issues.apache.org/jira/browse/CLOUDSTACK-6892) | Create separate package for the mysql HA component
 [CLOUDSTACK-7038](https://issues.apache.org/jira/browse/CLOUDSTACK-7038) | Add mysql client dependency for mgmt server/rpms
@@ -449,12 +447,12 @@ Bug ID | Description
 [CLOUDSTACK-7006](https://issues.apache.org/jira/browse/CLOUDSTACK-7006) | Restore template ID in ROOT volume usages
 [CLOUDSTACK-6747](https://issues.apache.org/jira/browse/CLOUDSTACK-6747) | test to allow all cidrs on other end of vpc
 [CLOUDSTACK-6272](https://issues.apache.org/jira/browse/CLOUDSTACK-6272) | Fix recover/restore VM actions
-[CLOUDSTACK-6927](https://issues.apache.org/jira/browse/CLOUDSTACK-6927) | store virsh list in list instead of querying libvirt 
+[CLOUDSTACK-6927](https://issues.apache.org/jira/browse/CLOUDSTACK-6927) | store virsh list in list instead of querying libvirt
 [CLOUDSTACK-6317](https://issues.apache.org/jira/browse/CLOUDSTACK-6317) | [VMware] Tagged VLAN support broken for Management/Control/Storage traffic
 [CLOUDSTACK-5891](https://issues.apache.org/jira/browse/CLOUDSTACK-5891) | [VMware] If a template has been registered and "cpu.corespersocket=X" ,
 [CLOUDSTACK-6478](https://issues.apache.org/jira/browse/CLOUDSTACK-6478) | Failed to download Template when having 3 SSVM's in one
 [CLOUDSTACK-6464](https://issues.apache.org/jira/browse/CLOUDSTACK-6464) | if guest network type is vlan://untagged, and traffic label is used
-[CLOUDSTACK-6816](https://issues.apache.org/jira/browse/CLOUDSTACK-6816) | bugfix: cloudstack-setup-management make /root directory's permission 0777 
+[CLOUDSTACK-6816](https://issues.apache.org/jira/browse/CLOUDSTACK-6816) | bugfix: cloudstack-setup-management make /root directory's permission 0777
 [CLOUDSTACK-6204](https://issues.apache.org/jira/browse/CLOUDSTACK-6204) | applying missed patch
 [CLOUDSTACK-6472](https://issues.apache.org/jira/browse/CLOUDSTACK-6472) | (4.3 specific) listUsageRecords: Pull information from removed items as well
 [CLOUDSTACK-5976](https://issues.apache.org/jira/browse/CLOUDSTACK-5976) | Typo in "ssh_keypairs" table's foreign key constraints on the Upgraded Setup
@@ -657,11 +655,11 @@ Version 4.1.0
 -------------
 
 This is the second major release of CloudStack from within the Apache Software Foundation, and the
-first major release as a Top-Level Project (TLP). 
+first major release as a Top-Level Project (TLP).
 
 Build Tool Changes:
 
- * The project now uses Maven 3 exclusively to build. 
+ * The project now uses Maven 3 exclusively to build.
 
 New Features:
 * CLOUDSTACK-101: OVS support in KVM
@@ -930,7 +928,6 @@ Security Fixes:
 
  * CVE-2012-4501: Apache CloudStack configuration vulnerability
 
-
 Version 4.0.2
 ------------------------
 
@@ -976,14 +973,13 @@ Issues fixed in this release:
 * CLOUDSTACK-1845: KVM - storage migration often fails
 * CLOUDSTACK-1846: KVM - storage pools can silently fail to be unregistered, leading to failure to register later
 * CLOUDSTACK-2003: Deleting domain while deleted account is cleaning up leaves VMs expunging forever due to 'Failed to update resource count'
-* CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1. 
+* CLOUDSTACK-2090: Upgrade from version 4.0.1 to version 4.0.2 triggers the 4.0.0 to 4.0.1.
 * CLOUDSTACK-2091: Error in API documentation for 4.0.x.
 
-
 Version 4.0.1-incubating
 ------------------------
 
-This is a bugfix release for Apache CloudStack 4.0.0-incubating, with no new features. 
+This is a bugfix release for Apache CloudStack 4.0.0-incubating, with no new features.
 
 Security Fixes:
 
@@ -1023,7 +1019,6 @@ Bugs fixed in this release:
 * CLOUDSTACK-961: Installation docs don't detail dependencies for building RPMs
 * CLOUDSTACK-995: Not able to add the KVM host
 
-
 Version 4.0.0-incubating
 ------------------------
 
@@ -1056,7 +1051,6 @@ Security Fixes:
 
  * CVE-2012-4501: Apache CloudStack configuration vulnerability
 
-
 Updating this file
 ------------------
 

CONTRIBUTING.md

@@ -3,44 +3,44 @@ Contributing to Apache CloudStack (ACS)
 
 Summary
 -------
-This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions. 
-These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project and you will submit a Pull Request for your changes to be added.
+This document covers how to contribute to the ACS project. ACS uses GitHub PRs to manage code contributions.
+These instructions assume you have a GitHub.com account, so if you don't have one you will have to create one. Your proposed code changes will be published to your own fork of the ACS project, and you will submit a Pull Request for your changes to be added.
 
-_Lets get started!!!_
+_Let's get started!!!_
 
 Bug fixes
 ---------
 
-It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches. 
-Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch. 
-This can be either the "current release" or the "previous release", depending on which ones are maintained. 
+It's very important that we can easily track bug fix commits, so their hashes should remain the same in all branches.
+Therefore, a pull request (PR) that fixes a bug, should be sent against a release branch.
+This can be either the "current release" or the "previous release", depending on which ones are maintained.
 Since the goal is a stable main, bug fixes should be "merged forward" to the next branch in order: "previous release" -> "current release" -> main (in other words: old to new)
 
 Developing new features
 -----------------------
 
-Development should be done in a feature branch, branched off of main. 
-Send a PR(steps below) to get it into main (2x LGTM applies). 
-PR will only be merged when main is open, will be held otherwise until main is open again. 
+Development should be done in a feature branch, branched off of main.
+Send a PR(steps below) to get it into main (2x LGTM applies).
+PR will only be merged when main is open, will be held otherwise until main is open again.
 No back porting / cherry-picking features to existing branches!
 
 PendingReleaseNotes file
 ------------------------
-When developing a new feature or making a (major) change to a existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
+When developing a new feature or making a (major) change to an existing feature you are encouraged to append this to the PendingReleaseNotes file so that the Release Manager can
 use this file as a source of information when compiling the Release Notes for a new release.
 
 When adding information to the PendingReleaseNotes file make sure that you write a good and understandable description of the new feature or change which you have developed.
 
 Updating the PendingReleaseNotes file is preferably a part of the original Pull Request, but that is up to the developers' discretion.
 
-Fork the code 
+Fork the code
 -------------
 
 In your browser, navigate to: [https://github.com/apache/cloudstack](https://github.com/apache/cloudstack)
 
-Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
+Fork the repository by clicking on the 'Fork' button on the top right hand side.  The fork will happen, and you will be taken to your own fork of the repository.  Copy the Git repository URL by clicking on the clipboard next to the URL on the right hand side of the page under '**HTTPS** clone URL'.  You will paste this URL when doing the following `git clone` command.
 
-On your computer, follow these steps to setup a local repository for working on ACS:
+On your computer, follow these steps to set up a local repository for working on ACS:
 
 ```bash
 $ git clone https://github.com/YOUR_ACCOUNT/cloudstack.git
@@ -51,11 +51,9 @@ $ git fetch upstream
 $ git rebase upstream/main
 ```
 
-
 Making changes
 --------------
 
-
 It is important that you create a new branch to make changes on and that you do not change the `main` branch (other than to rebase in changes from `upstream/main`).  In this example I will assume you will be making your changes to a branch called `feature_x`.  This `feature_x` branch will be created on your local repository and will be pushed to your forked repository on GitHub.  Once this branch is on your fork you will create a Pull Request for the changes to be added to the ACS project.
 
 It is best practice to create a new branch each time you want to contribute to the project and only track the changes for that pull request in this branch.
@@ -70,7 +68,6 @@ $ git commit -a -m "descriptive commit message for your changes"
 
 > The `-b` specifies that you want to create a new branch called `feature_x`.  You only specify `-b` the first time you checkout because you are creating a new branch.  Once the `feature_x` branch exists, you can later switch to it with only `git checkout feature_x`.
 
-
 Rebase `feature_x` to include updates from `upstream/main`
 ------------------------------------------------------------
 
@@ -92,13 +89,12 @@ $ git rebase main
 
 > Now your `feature_x` branch is up-to-date with all the code in `upstream/main`.
 
-
 Make a GitHub Pull Request to contribute your changes
 -----------------------------------------------------
 
-When you are happy with your changes and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
+When you are happy with your changes, and you are ready to contribute them, you will create a Pull Request on GitHub to do so.  This is done by pushing your local changes to your forked repository (default remote name is `origin`) and then initiating a pull request on GitHub.
 
-Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. Incase of UI PRs, a screenshot is preferred.
+Please include JIRA id, detailed information about the bug/feature, what all tests are executed, how the reviewer can test this feature etc. In case of UI PRs, a screenshot is preferred.
 
 > **IMPORTANT:** Make sure you have rebased your `feature_x` branch to include the latest code from `upstream/main` _before_ you do this.
 
@@ -107,7 +103,7 @@ $ git push origin main
 $ git push origin feature_x
 ```
 
-Now that the `feature_x` branch has been pushed to your GitHub repository, you can initiate the pull request.  
+Now that the `feature_x` branch has been pushed to your GitHub repository, you can initiate the pull request.
 
 To initiate the pull request, do the following:
 
@@ -118,7 +114,6 @@ To initiate the pull request, do the following:
 
 If you are requested to make modifications to your proposed changes, make the changes locally on your `feature_x` branch, re-push the `feature_x` branch to your fork.  The existing pull request should automatically pick up the change and update accordingly.
 
-
 Cleaning up after a successful pull request
 -------------------------------------------
 
@@ -136,4 +131,4 @@ $ git push origin :feature_x
 
 Release Principles
 ------------------
-Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up 
+Detailed information about ACS release principles is available at https://cwiki.apache.org/confluence/display/CLOUDSTACK/Release+principles+for+Apache+CloudStack+4.6+and+up

INSTALL.md

@@ -18,25 +18,26 @@ Install tools and dependencies used for development:
     # yum -y install git java-17-openjdk java-17-openjdk-devel \
       mysql mysql-server mkisofs git gcc python MySQL-python openssh-clients wget
 
-Set up Maven (3.6.0):
+Set up Maven (3.9.10):
 
-    # wget http://www.us.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
-    # tar -zxvf apache-maven-3.6.3-bin.tar.gz -C /usr/local
+    # wget https://dlcdn.apache.org/maven/maven-3/3.9.10/binaries/apache-maven-3.9.10-bin.tar.gz
+    # sudo tar -zxvf apache-maven-3.9.10-bin.tar.gz -C /usr/local
     # cd /usr/local
-    # ln -s apache-maven-3.6.3 maven
+    # sudo ln -s apache-maven-3.9.10 maven
     # echo export M2_HOME=/usr/local/maven >> ~/.bashrc # or .zshrc or .profile
     # echo export PATH=/usr/local/maven/bin:${PATH} >> ~/.bashrc # or .zshrc or .profile
     # source ~/.bashrc
 
-Setup up NodeJS (LTS):
+Setup up Node.js 16:
 
-    # curl -sL https://rpm.nodesource.com/setup_12.x | sudo bash -
+    # curl -sL https://rpm.nodesource.com/setup_16.x | sudo -E bash -
     # sudo yum install nodejs
     # sudo npm install -g @vue/cli npm-check-updates
 
 Start the MySQL service:
 
     $ service mysqld start
+    $ mysql_secure_installation
 
 ### Using jenv and/or pyenv for Version Management
 
@@ -78,7 +79,7 @@ Clear old database (if any) and deploy the database schema:
 
 Export the following variable if you need to run and debug the management server:
 
-    $ export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=500m -Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n"
+    $ export MAVEN_OPTS="-Xmx1024m -XX:MaxMetaspaceSize=500m -Xdebug -Xrunjdwp:transport=dt_socket,address=8787,server=y,suspend=n"
 
 Start the management server:
 
@@ -86,13 +87,33 @@ Start the management server:
 
 If this works, you've successfully setup a single server Apache CloudStack installation.
 
-Open the following URL on your browser to access the Management Server UI:
-
-    http://localhost:8080/client/
+To access the Management Server UI, follow the following procedure:
 
 The default credentials are; user: admin, password: password and the domain
 field should be left blank which is defaulted to the ROOT domain.
 
+## To bring up CloudStack UI
+
+Move to UI Directory
+
+    $ cd /path/to/cloudstack/ui
+
+To install dependencies.
+
+    $ npm install
+
+To build the project.
+
+    $ npm run build
+
+For Development Mode.
+
+    $ npm start
+
+Make sure to set `CS_URL=http://localhost:8080` on the `.env.local` file on UI.
+
+You should be able to run the management server on http://localhost:5050
+
 ## Building with non-redistributable plugins
 
 CloudStack supports several plugins that depend on libraries with distribution restrictions.

ISSUE_TEMPLATE.md

@@ -35,17 +35,14 @@ New line separated list of affected versions, commit ID for issues on main branc
 Information about the configuration if relevant, e.g. basic network, advanced networking, etc.  N/A otherwise
 -->
 
-
 ##### OS / ENVIRONMENT
 <!--
 Information about the environment if relevant, N/A otherwise
 -->
 
-
 ##### SUMMARY
 <!-- Explain the problem/feature briefly -->
 
-
 ##### STEPS TO REPRODUCE
 <!--
 For bugs, show exactly how to reproduce the problem, using a minimal test-case. Use Screenshots if accurate.

PULL_REQUEST_TEMPLATE.md

@@ -23,6 +23,7 @@ This PR...
 - [ ] Enhancement (improves an existing feature and functionality)
 - [ ] Cleanup (Code refactoring and cleanup, that may add test cases)
 - [ ] build/CI
+- [ ] test (unit or integration test code)
 
 ### Feature/Enhancement Scale or Bug Severity
 
@@ -39,10 +40,8 @@ This PR...
 - [ ] Minor
 - [ ] Trivial
 
-
 ### Screenshots (if appropriate):
 
-
 ### How Has This Been Tested?
 
 <!-- Please describe in detail how you tested your changes. -->
@@ -52,5 +51,4 @@ This PR...
 
 <!-- see how your change affects other areas of the code, etc. -->
 
-
 <!-- Please read the [CONTRIBUTING](https://github.com/apache/cloudstack/blob/main/CONTRIBUTING.md) document -->

README.md

@@ -142,7 +142,7 @@ This distribution includes cryptographic software. The country in which you curr
 reside may have restrictions on the import, possession, use, and/or re-export to another
 country, of encryption software. BEFORE using any encryption software, please check your
 country's laws, regulations and policies concerning the import, possession, or use, and
-re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/) 
+re-export of encryption software, to see if this is permitted. See [The Wassenaar Arrangement](http://www.wassenaar.org/)
 for more information.
 
 The U.S. Government Department of Commerce, Bureau of Industry and Security (BIS), has
@@ -160,3 +160,11 @@ The following provides more details on the included cryptographic software:
 * CloudStack makes use of the Bouncy Castle general-purpose encryption library.
 * CloudStack can optionally interact with and control OpenSwan-based VPNs.
 * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation.
+
+## Star History
+
+[![Apache CloudStack Star History](https://api.star-history.com/svg?repos=apache/cloudstack&type=Date)](https://www.star-history.com/#apache/cloudstack&Date)
+
+## Contributors
+
+[![Apache CloudStack Contributors](https://contrib.rocks/image?repo=apache/cloudstack&anon=0&max=500)](https://github.com/apache/cloudstack/graphs/contributors)

agent/bindir/cloud-setup-agent.in

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -20,6 +20,19 @@ import os
 import logging
 import sys
 import socket
+
+# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
+# ---- We do this so cloud_utils can be looked up in the following order:
+# ---- 1) Sources directory
+# ---- 2) waf configured PYTHONDIR
+# ---- 3) System Python path
+for pythonpath in (
+        "@PYTHONDIR@",
+        os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
+    ):
+        if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
+# ---- End snippet of code ----
+
 from cloudutils.cloudException import CloudRuntimeException, CloudInternalException
 from cloudutils.utilities import initLoging, bash
 from cloudutils.configFileOps import  configFileOps

agent/bindir/cloud-ssh.in

@@ -6,9 +6,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

agent/bindir/libvirtqemuhook.in

@@ -20,6 +20,19 @@ import sys
 import os
 import subprocess
 from threading import Timer
+
+# ---- This snippet of code adds the sources path and the waf configured PYTHONDIR to the Python path ----
+# ---- We do this so cloud_utils can be looked up in the following order:
+# ---- 1) Sources directory
+# ---- 2) waf configured PYTHONDIR
+# ---- 3) System Python path
+for pythonpath in (
+        "@PYTHONDIR@",
+        os.path.join(os.path.dirname(__file__),os.path.pardir,os.path.pardir,"python","lib"),
+    ):
+        if os.path.isdir(pythonpath): sys.path.insert(0,pythonpath)
+# ---- End snippet of code ----
+
 from xml.dom.minidom import parse
 from cloudutils.configFileOps import configFileOps
 from cloudutils.networkConfig import networkConfig

agent/conf/agent.properties

@@ -209,7 +209,7 @@ hypervisor.type=kvm
 # the management server would send.
 # In case of arm64 (aarch64), this will change the machine type to 'virt' and
 # adds a SCSI and a USB controller in the domain xml.
-# Possible values: x86_64 | aarch64
+# Possible values: x86_64 | aarch64 | s390x
 # If null (default), defaults to the VM's OS architecture
 #guest.cpu.arch=
 
@@ -286,6 +286,7 @@ hypervisor.type=kvm
 
 # The model of Watchdog timer to present to the Guest.
 # For all models refer to the libvirt documentation.
+# PLEASE NOTE: to disable the watchdogs definitions, use value: none
 #vm.watchdog.model=i6300esb
 
 # Action to take when the Guest/Instance is no longer notifying the Watchdog timer.
@@ -430,3 +431,22 @@ iscsi.session.cleanup.enabled=false
 # If set to "true", the agent will register for libvirt domain events, allowing for immediate updates on crashed or
 # unexpectedly stopped. Experimental, requires agent restart.
 # libvirt.events.enabled=false
+
+# Implicit host tags managed by agent.properties
+# host.tags=
+
+# Timeout(in seconds) for SSL handshake when agent connects to server. When no value is set then default value of 30s
+# will be used
+#ssl.handshake.timeout=
+
+# Wait(in seconds) during agent reconnections. When no value is set then default value of 5s will be used
+#backoff.seconds=
+
+# Timeout (in seconds) to wait for the snapshot reversion to complete.
+# revert.snapshot.timeout=10800
+
+# Timeout (in seconds) to wait for the incremental snapshot to complete.
+# incremental.snapshot.timeout=10800
+
+# If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise.
+# create.full.clone=false

agent/conf/cloudstack-agent.logrotate.in

@@ -15,11 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log {
+/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log /var/log/cloudstack/agent/agent.out /var/log/cloudstack/agent/agent.err {
     copytruncate
     daily
     rotate 5
     compress
     missingok
     size 10M
+    dateext
+    dateformat -%Y-%m-%d
 }

agent/conf/environment.properties.in

@@ -5,9 +5,9 @@
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
-# 
+#
 #   http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

agent/conf/log4j-cloud.xml.in

@@ -30,7 +30,7 @@ under the License.
          <Policies>
             <TimeBasedTriggeringPolicy/>
          </Policies>
-         <PatternLayout pattern="%d{ISO8601} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
+         <PatternLayout pattern="%d{DEFAULT} %-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
       </RollingFile>
 
       <!-- ============================== -->
@@ -38,7 +38,7 @@ under the License.
       <!-- ============================== -->
 
       <Console name="CONSOLE" target="SYSTEM_OUT">
-          <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+          <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
           <PatternLayout pattern="%-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
       </Console>
    </Appenders>

agent/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -16,29 +16,6 @@
 // under the License.
 package com.cloud.agent;
 
-import com.cloud.agent.Agent.ExitStatus;
-import com.cloud.agent.dao.StorageComponent;
-import com.cloud.agent.dao.impl.PropertiesStorage;
-import com.cloud.agent.properties.AgentProperties;
-import com.cloud.agent.properties.AgentPropertiesFileHandler;
-import com.cloud.resource.ServerResource;
-import com.cloud.utils.LogUtils;
-import com.cloud.utils.ProcessUtil;
-import com.cloud.utils.PropertiesUtil;
-import com.cloud.utils.backoff.BackoffAlgorithm;
-import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
-import com.cloud.utils.exception.CloudRuntimeException;
-import org.apache.commons.daemon.Daemon;
-import org.apache.commons.daemon.DaemonContext;
-import org.apache.commons.daemon.DaemonInitException;
-import org.apache.commons.lang.math.NumberUtils;
-import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.core.config.Configurator;
-
-import javax.naming.ConfigurationException;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -53,6 +30,31 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.UUID;
 
+import javax.naming.ConfigurationException;
+
+import org.apache.commons.daemon.Daemon;
+import org.apache.commons.daemon.DaemonContext;
+import org.apache.commons.daemon.DaemonInitException;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.config.Configurator;
+
+import com.cloud.agent.Agent.ExitStatus;
+import com.cloud.agent.dao.StorageComponent;
+import com.cloud.agent.dao.impl.PropertiesStorage;
+import com.cloud.agent.properties.AgentProperties;
+import com.cloud.agent.properties.AgentPropertiesFileHandler;
+import com.cloud.resource.ServerResource;
+import com.cloud.utils.LogUtils;
+import com.cloud.utils.ProcessUtil;
+import com.cloud.utils.PropertiesUtil;
+import com.cloud.utils.backoff.BackoffAlgorithm;
+import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
+import com.cloud.utils.exception.CloudRuntimeException;
+
 public class AgentShell implements IAgentShell, Daemon {
     protected static Logger LOGGER = LogManager.getLogger(AgentShell.class);
 
@@ -64,6 +66,7 @@ public class AgentShell implements IAgentShell, Daemon {
     private String _zone;
     private String _pod;
     private String _host;
+    private List<String> _avoidHosts;
     private String _privateIp;
     private int _port;
     private int _proxyPort;
@@ -74,9 +77,9 @@ public class AgentShell implements IAgentShell, Daemon {
     private volatile boolean _exit = false;
     private int _pingRetries;
     private final List<Agent> _agents = new ArrayList<Agent>();
-    private String hostToConnect;
     private String connectedHost;
     private Long preferredHostCheckInterval;
+    private boolean connectionTransfer = false;
     protected AgentProperties agentProperties = new AgentProperties();
 
     public AgentShell() {
@@ -118,7 +121,7 @@ public class AgentShell implements IAgentShell, Daemon {
         if (_hostCounter >= hosts.length) {
             _hostCounter = 0;
         }
-        hostToConnect = hosts[_hostCounter % hosts.length];
+        String hostToConnect = hosts[_hostCounter % hosts.length];
         _hostCounter++;
         return hostToConnect;
     }
@@ -140,11 +143,10 @@ public class AgentShell implements IAgentShell, Daemon {
     }
 
     @Override
-    public void updateConnectedHost() {
-        connectedHost = hostToConnect;
+    public void updateConnectedHost(String connectedHost) {
+        this.connectedHost = connectedHost;
     }
 
-
     @Override
     public void resetHostCounter() {
         _hostCounter = 0;
@@ -163,6 +165,16 @@ public class AgentShell implements IAgentShell, Daemon {
         }
     }
 
+    @Override
+    public void setAvoidHosts(List<String> avoidHosts) {
+        _avoidHosts = avoidHosts;
+    }
+
+    @Override
+    public List<String> getAvoidHosts() {
+        return _avoidHosts;
+    }
+
     @Override
     public String getPrivateIp() {
         return _privateIp;
@@ -215,6 +227,14 @@ public class AgentShell implements IAgentShell, Daemon {
             _storage.persist(name, value);
     }
 
+    public boolean isConnectionTransfer() {
+        return connectionTransfer;
+    }
+
+    public void setConnectionTransfer(boolean connectionTransfer) {
+        this.connectionTransfer = connectionTransfer;
+    }
+
     void loadProperties() throws ConfigurationException {
         final File file = PropertiesUtil.findConfigFile("agent.properties");
 
@@ -406,7 +426,9 @@ public class AgentShell implements IAgentShell, Daemon {
 
         LOGGER.info("Defaulting to the constant time backoff algorithm");
         _backoff = new ConstantTimeBackoff();
-        _backoff.configure("ConstantTimeBackoff", new HashMap<String, Object>());
+        Map<String, Object> map = new HashMap<>();
+        map.put("seconds", _properties.getProperty("backoff.seconds"));
+        _backoff.configure("ConstantTimeBackoff", map);
     }
 
     private void launchAgent() throws ConfigurationException {
@@ -455,6 +477,11 @@ public class AgentShell implements IAgentShell, Daemon {
         agent.start();
     }
 
+    @Override
+    public Integer getSslHandshakeTimeout() {
+        return AgentPropertiesFileHandler.getPropertyValue(AgentProperties.SSL_HANDSHAKE_TIMEOUT);
+    }
+
     public synchronized int getNextAgentId() {
         return _nextAgentId++;
     }

agent/src/main/java/com/cloud/agent/IAgentShell.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.agent;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 
@@ -63,11 +64,21 @@ public interface IAgentShell {
 
     String[] getHosts();
 
+    void setAvoidHosts(List<String> hosts);
+
+    List<String> getAvoidHosts();
+
     long getLbCheckerInterval(Long receivedLbInterval);
 
-    void updateConnectedHost();
+    void updateConnectedHost(String connectedHost);
 
     String getConnectedHost();
 
     void launchNewAgent(ServerResource resource) throws ConfigurationException;
+
+    boolean isConnectionTransfer();
+
+    void setConnectionTransfer(boolean connectionTransfer);
+
+    Integer getSslHandshakeTimeout();
 }

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -155,6 +155,14 @@ public class AgentProperties{
      */
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
+    /**
+     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * This configuration is only considered if libvirt.events.enabled is also true. <br>
+     * Data type: Integer.<br>
+     * Default value: <code>259200</code>
+     */
+    public static final Property<Integer> QCOW2_DELTA_MERGE_TIMEOUT = new Property<>("qcow2.delta.merge.timeout", 60 * 60 * 72);
+
     /**
      * This parameter sets the VM migration speed (in mbps). The default value is -1,<br>
      * which means that the agent will try to guess the speed of the guest network and consume all possible bandwidth.<br>
@@ -213,6 +221,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.script", "libvirt-vm-xml-transformer.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM XML transformer shell script.<br>
+     * The shell script is used to execute the Libvirt VM XML transformer script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-xml-transformer.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_XML_TRANSFORMER_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_xml_transformer.shell_script", "libvirt-vm-xml-transformer.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_xml_transformer.script properties to define the Libvirt VM XML transformer method.<br>
      * Libvirt XML transformer hook does XML-to-XML transformation.<br>
@@ -233,6 +250,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on start shell script.<br>
+     * The shell script is used to execute the Libvirt VM on start script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_START_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_start.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_start.script properties to define the Libvirt VM on start method.<br>
      * The hook is called right after Libvirt successfully launched the VM.<br>
@@ -252,6 +278,15 @@ public class AgentProperties{
      */
     public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.script", "libvirt-vm-state-change.groovy");
 
+    /**
+     * This property is used with the agent.hooks.basedir property to define the Libvirt VM on stop shell script.<br>
+     * The shell script is used to execute the Libvirt VM on stop script.<br>
+     * For more information see the agent.properties file.<br>
+     * Data type: String.<br>
+     * Default value: <code>libvirt-vm-state-change.sh</code>
+     */
+    public static final Property<String> AGENT_HOOKS_LIBVIRT_VM_ON_STOP_SHELL_SCRIPT = new Property<>("agent.hooks.libvirt_vm_on_stop.shell_script", "libvirt-vm-state-change.sh");
+
     /**
      * This property is used with the agent.hooks.basedir and agent.hooks.libvirt_vm_on_stop.script properties to define the Libvirt VM on stop method.<br>
      * The hook is called right after libvirt successfully stopped the VM.<br>
@@ -383,7 +418,7 @@ public class AgentProperties{
     /**
      * This param will set the CPU architecture for the domain to override what the management server would send.<br>
      * In case of arm64 (aarch64), this will change the machine type to 'virt' and add a SCSI and a USB controller in the domain XML.<br>
-     * Possible values: x86_64 | aarch64 <br>
+     * Possible values: x86_64 | aarch64 | s390x <br>
      * Data type: String.<br>
      * Default value: <code>null</code> (will set use the architecture of the VM's OS).
      */
@@ -516,6 +551,7 @@ public class AgentProperties{
     /**
      * The model of Watchdog timer to present to the Guest.<br>
      * For all models refer to the libvirt documentation.<br>
+     * PLEASE NOTE: to disable the watchdogs definitions, use value: none
      * Data type: String.<br>
      * Default value: <code>i6300esb</code>
      */
@@ -751,7 +787,7 @@ public class AgentProperties{
     public static final Property<Integer> IOTHREADS = new Property<>("iothreads", 1);
 
     /**
-     * Enable verbose mode for virt-v2v Instance Conversion from Vmware to KVM
+     * Enable verbose mode for virt-v2v Instance Conversion from VMware to KVM
      * Data type: Boolean.<br>
      * Default value: <code>false</code>
      */
@@ -803,12 +839,44 @@ public class AgentProperties{
      */
     public static final Property<String> KEYSTORE_PASSPHRASE = new Property<>(KeyStoreUtils.KS_PASSPHRASE_PROPERTY, null, String.class);
 
+    /**
+     * Implicit host tags
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
+
+    /**
+     * Timeout for SSL handshake in seconds
+     * Data type: Integer.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<Integer> SSL_HANDSHAKE_TIMEOUT = new Property<>("ssl.handshake.timeout", 30, Integer.class);
+
+    /**
+     * Timeout (in seconds) to wait for the incremental snapshot to complete.
+     * */
+    public static final Property<Integer> INCREMENTAL_SNAPSHOT_TIMEOUT = new Property<>("incremental.snapshot.timeout", 10800);
+
+    /**
+     * Timeout (in seconds) to wait for the snapshot reversion to complete.
+     * */
+    public static final Property<Integer> REVERT_SNAPSHOT_TIMEOUT = new Property<>("revert.snapshot.timeout", 10800);
+
+    /**
+     *  If set to true, creates VMs as full clones of their templates on KVM hypervisor. Creates as linked clones otherwise. <br>
+     * Data type: Boolean. <br>
+     * Default value: <code>false</code>
+     */
+    public static final Property<Boolean> CREATE_FULL_CLONE = new Property<>("create.full.clone", false);
+
+
     public static class Property <T>{
         private String name;
         private T defaultValue;
         private Class<T> typeClass;
 
-        Property(String name, T value) {
+        public Property(String name, T value) {
             init(name, value);
         }
 

agent/src/main/java/com/cloud/agent/resource/consoleproxy/ConsoleProxyResource.java

@@ -397,9 +397,8 @@ public class ConsoleProxyResource extends ServerResourceBase implements ServerRe
     }
 
     public String authenticateConsoleAccess(String host, String port, String vmId, String sid, String ticket,
-                                            Boolean isReauthentication, String sessionToken) {
-
-        ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken);
+                                            Boolean isReauthentication, String sessionToken, String clientAddress) {
+        ConsoleAccessAuthenticationCommand cmd = new ConsoleAccessAuthenticationCommand(host, port, vmId, sid, ticket, sessionToken, clientAddress);
         cmd.setReauthenticating(isReauthentication);
 
         ConsoleProxyAuthenticationResult result = new ConsoleProxyAuthenticationResult();

agent/src/test/java/com/cloud/agent/AgentShellTest.java

@@ -350,4 +350,23 @@ public class AgentShellTest {
 
         Mockito.verify(agentShellSpy).setHosts(expected);
     }
+
+    @Test
+    public void updateAndGetConnectedHost() {
+        String expected = "test";
+
+        AgentShell shell = new AgentShell();
+        shell.setHosts("test");
+        shell.getNextHost();
+        shell.updateConnectedHost("test");
+
+        Assert.assertEquals(expected, shell.getConnectedHost());
+    }
+
+    @Test
+    public void testGetSslHandshakeTimeout() {
+        Integer expected = 1;
+        agentPropertiesFileHandlerMocked.when(() -> AgentPropertiesFileHandler.getPropertyValue(Mockito.eq(AgentProperties.SSL_HANDSHAKE_TIMEOUT))).thenReturn(expected);
+        Assert.assertEquals(expected, agentShellSpy.getSslHandshakeTimeout());
+    }
 }

agent/src/test/java/com/cloud/agent/AgentTest.java

@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import javax.naming.ConfigurationException;
+
+import org.apache.logging.log4j.Logger;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import com.cloud.resource.ServerResource;
+import com.cloud.utils.backoff.impl.ConstantTimeBackoff;
+import com.cloud.utils.nio.Link;
+import com.cloud.utils.nio.NioConnection;
+
+@RunWith(MockitoJUnitRunner.class)
+public class AgentTest {
+    Agent agent;
+    private AgentShell shell;
+    private ServerResource serverResource;
+    private Logger logger;
+
+    @Before
+    public void setUp() throws ConfigurationException {
+        shell = mock(AgentShell.class);
+        serverResource = mock(ServerResource.class);
+        doReturn(true).when(serverResource).configure(any(), any());
+        doReturn(1).when(shell).getWorkers();
+        doReturn(1).when(shell).getPingRetries();
+        agent = new Agent(shell, 1, serverResource);
+        logger = mock(Logger.class);
+        ReflectionTestUtils.setField(agent, "logger", logger);
+    }
+
+    @Test
+    public void testGetLinkLogNullLinkReturnsEmptyString() {
+        Link link = null;
+        String result = agent.getLinkLog(link);
+        assertEquals("", result);
+    }
+
+    @Test
+    public void testGetLinkLogLinkWithTraceEnabledReturnsLinkLogWithHashCode() {
+        Link link = mock(Link.class);
+        InetSocketAddress socketAddress = new InetSocketAddress("192.168.1.100", 1111);
+        when(link.getSocketAddress()).thenReturn(socketAddress);
+        when(logger.isTraceEnabled()).thenReturn(true);
+
+        String result = agent.getLinkLog(link);
+        System.out.println(result);
+        assertTrue(result.startsWith(System.identityHashCode(link) + "-"));
+        assertTrue(result.contains("192.168.1.100"));
+    }
+
+    @Test
+    public void testGetAgentNameWhenServerResourceIsNull() {
+        ReflectionTestUtils.setField(agent, "serverResource", null);
+        assertEquals("Agent", agent.getAgentName());
+    }
+
+    @Test
+    public void testGetAgentNameWhenAppendAgentNameIsTrue() {
+        when(serverResource.isAppendAgentNameToLogs()).thenReturn(true);
+        when(serverResource.getName()).thenReturn("TestAgent");
+
+        String agentName = agent.getAgentName();
+        assertEquals("TestAgent", agentName);
+    }
+
+    @Test
+    public void testGetAgentNameWhenAppendAgentNameIsFalse() {
+        when(serverResource.isAppendAgentNameToLogs()).thenReturn(false);
+
+        String agentName = agent.getAgentName();
+        assertEquals("Agent", agentName);
+    }
+
+    @Test
+    public void testAgentInitialization() {
+        Runtime.getRuntime().removeShutdownHook(agent.shutdownThread);
+        when(shell.getPingRetries()).thenReturn(3);
+        when(shell.getWorkers()).thenReturn(5);
+        agent.setupShutdownHookAndInitExecutors();
+        assertNotNull(agent.selfTaskExecutor);
+        assertNotNull(agent.outRequestHandler);
+        assertNotNull(agent.requestHandler);
+    }
+
+    @Test
+    public void testAgentShutdownHookAdded() {
+        Runtime.getRuntime().removeShutdownHook(agent.shutdownThread);
+        agent.setupShutdownHookAndInitExecutors();
+        verify(logger).trace("Adding shutdown hook");
+    }
+
+    @Test
+    public void testGetResourceGuidValidGuidAndResourceName() {
+        when(shell.getGuid()).thenReturn("12345");
+        String result = agent.getResourceGuid();
+        assertTrue(result.startsWith("12345-" + ServerResource.class.getSimpleName()));
+    }
+
+    @Test
+    public void testGetZoneReturnsValidZone() {
+        when(shell.getZone()).thenReturn("ZoneA");
+        String result = agent.getZone();
+        assertEquals("ZoneA", result);
+    }
+
+    @Test
+    public void testGetPodReturnsValidPod() {
+        when(shell.getPod()).thenReturn("PodA");
+        String result = agent.getPod();
+        assertEquals("PodA", result);
+    }
+
+    @Test
+    public void testSetLinkAssignsLink() {
+        Link mockLink = mock(Link.class);
+        agent.setLink(mockLink);
+        assertEquals(mockLink, agent.link);
+    }
+
+    @Test
+    public void testGetResourceReturnsServerResource() {
+        ServerResource mockResource = mock(ServerResource.class);
+        ReflectionTestUtils.setField(agent, "serverResource", mockResource);
+        ServerResource result = agent.getResource();
+        assertSame(mockResource, result);
+    }
+
+    @Test
+    public void testGetResourceName() {
+        String result = agent.getResourceName();
+        assertTrue(result.startsWith(ServerResource.class.getSimpleName()));
+    }
+
+    @Test
+    public void testUpdateLastPingResponseTimeUpdatesCurrentTime() {
+        long beforeUpdate = System.currentTimeMillis();
+        agent.updateLastPingResponseTime();
+        long updatedTime = agent.lastPingResponseTime.get();
+        assertTrue(updatedTime >= beforeUpdate);
+        assertTrue(updatedTime <= System.currentTimeMillis());
+    }
+
+    @Test
+    public void testGetNextSequenceIncrementsSequence() {
+        long initialSequence = agent.getNextSequence();
+        long nextSequence = agent.getNextSequence();
+        assertEquals(initialSequence + 1, nextSequence);
+        long thirdSequence = agent.getNextSequence();
+        assertEquals(nextSequence + 1, thirdSequence);
+    }
+
+    @Test
+    public void testRegisterControlListenerAddsListener() {
+        IAgentControlListener listener = mock(IAgentControlListener.class);
+        agent.registerControlListener(listener);
+        assertTrue(agent.controlListeners.contains(listener));
+    }
+
+    @Test
+    public void testUnregisterControlListenerRemovesListener() {
+        IAgentControlListener listener = mock(IAgentControlListener.class);
+        agent.registerControlListener(listener);
+        assertTrue(agent.controlListeners.contains(listener));
+        agent.unregisterControlListener(listener);
+        assertFalse(agent.controlListeners.contains(listener));
+    }
+
+    @Test
+    public void testCloseAndTerminateLinkLinkIsNullDoesNothing() {
+        agent.closeAndTerminateLink(null);
+    }
+
+    @Test
+    public void testCloseAndTerminateLinkValidLinkCallsCloseAndTerminate() {
+        Link mockLink = mock(Link.class);
+        agent.closeAndTerminateLink(mockLink);
+        verify(mockLink).close();
+        verify(mockLink).terminated();
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionConnectionIsNullDoesNothing() {
+        agent.connection = null;
+        agent.stopAndCleanupConnection(false);
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionValidConnectionNoWaitStopsAndCleansUp() throws IOException {
+        NioConnection mockConnection = mock(NioConnection.class);
+        agent.connection = mockConnection;
+        agent.stopAndCleanupConnection(false);
+        verify(mockConnection).stop();
+        verify(mockConnection).cleanUp();
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionCleanupThrowsIOExceptionLogsWarning() throws IOException {
+        NioConnection mockConnection = mock(NioConnection.class);
+        agent.connection = mockConnection;
+        doThrow(new IOException("Cleanup failed")).when(mockConnection).cleanUp();
+        agent.stopAndCleanupConnection(false);
+        verify(mockConnection).stop();
+        verify(logger).warn(eq("Fail to clean up old connection. {}"), any(IOException.class));
+    }
+
+    @Test
+    public void testStopAndCleanupConnectionValidConnectionWaitForStopWaitsForStartupToStop() throws IOException {
+        NioConnection mockConnection = mock(NioConnection.class);
+        ConstantTimeBackoff mockBackoff = mock(ConstantTimeBackoff.class);
+        mockBackoff.setTimeToWait(0);
+        agent.connection = mockConnection;
+        when(shell.getBackoffAlgorithm()).thenReturn(mockBackoff);
+        when(mockConnection.isStartup()).thenReturn(true, true, false);
+        agent.stopAndCleanupConnection(true);
+        verify(mockConnection).stop();
+        verify(mockConnection).cleanUp();
+        verify(mockBackoff, times(3)).waitBeforeRetry();
+    }
+}

api/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.cloudstack</groupId>
         <artifactId>cloudstack</artifactId>
-        <version>4.20.0.0-SNAPSHOT</version>
+        <version>4.21.0.0-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>

api/src/main/java/com/cloud/agent/api/Command.java

@@ -19,9 +19,10 @@ package com.cloud.agent.api;
 import java.util.HashMap;
 import java.util.Map;
 
-import com.cloud.agent.api.LogLevel.Log4jLevel;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.cloud.agent.api.LogLevel.Log4jLevel;
 
 /**
  * implemented by classes that extends the Command class. Command specifies
@@ -35,6 +36,23 @@ public abstract class Command {
         Continue, Stop
     }
 
+    public enum State {
+        CREATED,        // Command is created by management server
+        STARTED,        // Command is started by agent
+        PROCESSING,     // Processing by agent
+        PROCESSING_IN_BACKEND,  // Processing in backend by agent
+        COMPLETED,      // Operation succeeds by agent or management server
+        FAILED,         // Operation fails by agent
+        RECONCILE_RETRY,        // Ready for retry of reconciliation
+        RECONCILING,    // Being reconciled by management server
+        RECONCILED,     // Reconciled by management server
+        RECONCILE_SKIPPED,  // Skip the reconciliation as the resource state is inconsistent with the command
+        RECONCILE_FAILED,       // Fail to reconcile by management server
+        TIMED_OUT,      // Timed out on management server or agent
+        INTERRUPTED,    // Interrupted by management server or agent (for example agent is restarted),
+        DANGLED_IN_BACKEND     // Backend process which cannot be processed normally (for example agent is restarted)
+    }
+
     public static final String HYPERVISOR_TYPE = "hypervisorType";
 
     // allow command to carry over hypervisor or other environment related context info
@@ -42,6 +60,8 @@ public abstract class Command {
     protected Map<String, String> contextMap = new HashMap<String, String>();
     private int wait;  //in second
     private boolean bypassHostMaintenance = false;
+    private transient long requestSequence = 0L;
+    protected Map<String, Map<String, String>> externalDetails;
 
     protected Command() {
         this.wait = 0;
@@ -82,6 +102,10 @@ public abstract class Command {
         return contextMap.get(name);
     }
 
+    public Map<String, String> getContextMap() {
+        return contextMap;
+    }
+
     public boolean allowCaching() {
         return true;
     }
@@ -94,6 +118,26 @@ public abstract class Command {
         this.bypassHostMaintenance = bypassHostMaintenance;
     }
 
+    public boolean isReconcile() {
+        return false;
+    }
+
+    public long getRequestSequence() {
+        return requestSequence;
+    }
+
+    public void setRequestSequence(long requestSequence) {
+        this.requestSequence = requestSequence;
+    }
+
+    public void setExternalDetails(Map<String, Map<String, String>> externalDetails) {
+        this.externalDetails = externalDetails;
+    }
+
+    public Map<String, Map<String, String>> getExternalDetails() {
+        return externalDetails;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

api/src/main/java/com/cloud/agent/api/VgpuTypesInfo.java

@@ -15,10 +15,24 @@
 // specific language governing permissions and limitations
 // under the License.
 package com.cloud.agent.api;
+
+import org.apache.cloudstack.gpu.GpuDevice;
+
 public class VgpuTypesInfo {
 
+    private boolean passthroughEnabled = true;
+    private GpuDevice.DeviceType deviceType;
+    private String parentBusAddress;
+    private String busAddress;
+    private String numaNode;
+    private String pciRoot;
+    private String deviceId;
+    private String deviceName;
+    private String vendorId;
+    private String vendorName;
     private String modelName;
     private String groupName;
+    private String vmName;
     private Long maxHeads;
     private Long videoRam;
     private Long maxResolutionX;
@@ -26,6 +40,7 @@ public class VgpuTypesInfo {
     private Long maxVgpuPerGpu;
     private Long remainingCapacity;
     private Long maxCapacity;
+    private boolean display = false;
 
     public String getModelName() {
         return modelName;
@@ -39,22 +54,42 @@ public class VgpuTypesInfo {
         return videoRam;
     }
 
+    public void setVideoRam(Long videoRam) {
+        this.videoRam = videoRam;
+    }
+
     public Long getMaxHeads() {
         return maxHeads;
     }
 
+    public void setMaxHeads(Long maxHeads) {
+        this.maxHeads = maxHeads;
+    }
+
     public Long getMaxResolutionX() {
         return maxResolutionX;
     }
 
+    public void setMaxResolutionX(Long maxResolutionX) {
+        this.maxResolutionX = maxResolutionX;
+    }
+
     public Long getMaxResolutionY() {
         return maxResolutionY;
     }
 
+    public void setMaxResolutionY(Long maxResolutionY) {
+        this.maxResolutionY = maxResolutionY;
+    }
+
     public Long getMaxVpuPerGpu() {
         return maxVgpuPerGpu;
     }
 
+    public void setMaxVgpuPerGpu(Long maxVgpuPerGpu) {
+        this.maxVgpuPerGpu = maxVgpuPerGpu;
+    }
+
     public Long getRemainingCapacity() {
         return remainingCapacity;
     }
@@ -71,8 +106,133 @@ public class VgpuTypesInfo {
         this.maxCapacity = maxCapacity;
     }
 
-    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX, Long maxResolutionY, Long maxVgpuPerGpu,
-            Long remainingCapacity, Long maxCapacity) {
+    public boolean isPassthroughEnabled() {
+        return passthroughEnabled;
+    }
+
+    public void setPassthroughEnabled(boolean passthroughEnabled) {
+        this.passthroughEnabled = passthroughEnabled;
+    }
+
+    public GpuDevice.DeviceType getDeviceType() {
+        return deviceType;
+    }
+
+    public void setDeviceType(GpuDevice.DeviceType deviceType) {
+        this.deviceType = deviceType;
+    }
+
+    public String getParentBusAddress() {
+        return parentBusAddress;
+    }
+
+    public void setParentBusAddress(String parentBusAddress) {
+        this.parentBusAddress = parentBusAddress;
+    }
+
+    public String getBusAddress() {
+        return busAddress;
+    }
+
+    public void setBusAddress(String busAddress) {
+        this.busAddress = busAddress;
+    }
+
+    public String getNumaNode() {
+        return numaNode;
+    }
+
+    public void setNumaNode(String numaNode) {
+        this.numaNode = numaNode;
+    }
+
+    public String getPciRoot() {
+        return pciRoot;
+    }
+
+    public void setPciRoot(String pciRoot) {
+        this.pciRoot = pciRoot;
+    }
+
+    public String getDeviceId() {
+        return deviceId;
+    }
+
+    public void setDeviceId(String deviceId) {
+        this.deviceId = deviceId;
+    }
+
+    public String getDeviceName() {
+        return deviceName;
+    }
+
+    public void setDeviceName(String deviceName) {
+        this.deviceName = deviceName;
+    }
+
+    public String getVendorId() {
+        return vendorId;
+    }
+
+    public void setVendorId(String vendorId) {
+        this.vendorId = vendorId;
+    }
+
+    public String getVendorName() {
+        return vendorName;
+    }
+
+    public void setVendorName(String vendorName) {
+        this.vendorName = vendorName;
+    }
+
+    public String getVmName() {
+        return vmName;
+    }
+
+    public void setVmName(String vmName) {
+        this.vmName = vmName;
+    }
+
+    public boolean isDisplay() {
+        return display;
+    }
+
+    public void setDisplay(boolean display) {
+        this.display = display;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+            String vendorId, String vendorName, String deviceId, String deviceName, String numaNode, String pciRoot
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+        this.numaNode = numaNode;
+        this.pciRoot = pciRoot;
+    }
+
+    public VgpuTypesInfo(GpuDevice.DeviceType deviceType, String groupName, String modelName, String busAddress,
+                         String vendorId, String vendorName, String deviceId, String deviceName
+    ) {
+        this.deviceType = deviceType;
+        this.groupName = groupName;
+        this.modelName = modelName;
+        this.busAddress = busAddress;
+        this.deviceId = deviceId;
+        this.deviceName = deviceName;
+        this.vendorId = vendorId;
+        this.vendorName = vendorName;
+    }
+
+    public VgpuTypesInfo(String groupName, String modelName, Long videoRam, Long maxHeads, Long maxResolutionX,
+            Long maxResolutionY, Long maxVgpuPerGpu, Long remainingCapacity, Long maxCapacity
+    ) {
         this.groupName = groupName;
         this.modelName = modelName;
         this.videoRam = videoRam;

api/src/main/java/com/cloud/agent/api/to/BucketTO.java

@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.agent.api.to;
+
+import org.apache.cloudstack.storage.object.Bucket;
+
+public final class BucketTO {
+
+    private String name;
+
+    private String accessKey;
+
+    private String secretKey;
+
+    public BucketTO(Bucket bucket) {
+        this.name = bucket.getName();
+        this.accessKey = bucket.getAccessKey();
+        this.secretKey = bucket.getSecretKey();
+    }
+
+    public BucketTO(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    public String getAccessKey() {
+        return this.accessKey;
+    }
+
+    public String getSecretKey() {
+        return this.secretKey;
+    }
+}

api/src/main/java/com/cloud/agent/api/to/DiskTO.java

@@ -46,7 +46,7 @@ public class DiskTO {
     private Long diskSeq;
     private String path;
     private Volume.Type type;
-    private Map<String, String> _details;
+    private Map<String, String> details;
 
     public DiskTO() {
 
@@ -92,10 +92,10 @@ public class DiskTO {
     }
 
     public void setDetails(Map<String, String> details) {
-        _details = details;
+        this.details = details;
     }
 
     public Map<String, String> getDetails() {
-        return _details;
+        return details;
     }
 }

api/src/main/java/com/cloud/agent/api/to/FirewallRuleTO.java

@@ -47,7 +47,7 @@ public class FirewallRuleTO implements InternalIdentity {
     int[] srcPortRange;
     boolean revoked;
     boolean alreadyAdded;
-    private List<String> sourceCidrList;
+    protected List<String> sourceCidrList;
     private List<String> destCidrList;
     FirewallRule.Purpose purpose;
     private Integer icmpType;
@@ -155,9 +155,7 @@ public class FirewallRuleTO implements InternalIdentity {
             rule.getIcmpType(),
             rule.getIcmpCode());
         this.trafficType = trafficType;
-        if (FirewallRule.Purpose.Ipv6Firewall.equals(purpose)) {
-            this.destCidrList = rule.getDestinationCidrList();
-        }
+        this.destCidrList = rule.getDestinationCidrList();
     }
 
     public FirewallRuleTO(FirewallRule rule, String srcVlanTag, String srcIp, FirewallRule.Purpose purpose, FirewallRule.TrafficType trafficType,

api/src/main/java/com/cloud/agent/api/to/GPUDeviceTO.java

@@ -16,7 +16,9 @@
 // under the License.
 package com.cloud.agent.api.to;
 
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 
 import com.cloud.agent.api.VgpuTypesInfo;
 
@@ -24,9 +26,23 @@ public class GPUDeviceTO {
 
     private String gpuGroup;
     private String vgpuType;
+    private int gpuCount;
     private HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails = new HashMap<String, HashMap<String, VgpuTypesInfo>>();
+    private List<VgpuTypesInfo> gpuDevices = new ArrayList<>();
 
-    public GPUDeviceTO( String gpuGroup, String vgpuType, HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
+    public GPUDeviceTO(String gpuGroup, String vgpuType, int gpuCount,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails,
+                       List<VgpuTypesInfo> gpuDevices) {
+        this.gpuGroup = gpuGroup;
+        this.vgpuType = vgpuType;
+        this.groupDetails = groupDetails;
+        this.gpuCount = gpuCount;
+        this.gpuDevices = gpuDevices;
+
+    }
+
+    public GPUDeviceTO(String gpuGroup, String vgpuType,
+                       HashMap<String, HashMap<String, VgpuTypesInfo>> groupDetails) {
         this.gpuGroup = gpuGroup;
         this.vgpuType = vgpuType;
         this.groupDetails = groupDetails;
@@ -48,6 +64,14 @@ public class GPUDeviceTO {
         this.vgpuType = vgpuType;
     }
 
+    public int getGpuCount() {
+        return gpuCount;
+    }
+
+    public void setGpuCount(int gpuCount) {
+        this.gpuCount = gpuCount;
+    }
+
     public HashMap<String, HashMap<String, VgpuTypesInfo>> getGroupDetails() {
         return groupDetails;
     }
@@ -56,4 +80,11 @@ public class GPUDeviceTO {
         this.groupDetails = groupDetails;
     }
 
+    public List<VgpuTypesInfo> getGpuDevices() {
+        return gpuDevices;
+    }
+
+    public void setGpuDevices(List<VgpuTypesInfo> gpuDevices) {
+        this.gpuDevices = gpuDevices;
+    }
 }

api/src/main/java/com/cloud/agent/api/to/LoadBalancerTO.java

@@ -374,13 +374,15 @@ public class LoadBalancerTO {
     public static class CounterTO implements Serializable {
         private static final long serialVersionUID = 2L;
         private final Long id;
+        private final String uuid;
         private final String name;
         private final Counter.Source source;
         private final String value;
         private final String provider;
 
-        public CounterTO(Long id, String name, Counter.Source source, String value, String provider) {
+        public CounterTO(Long id, String uuid, String name, Counter.Source source, String value, String provider) {
             this.id = id;
+            this.uuid = uuid;
             this.name = name;
             this.source = source;
             this.value = value;
@@ -391,6 +393,10 @@ public class LoadBalancerTO {
             return id;
         }
 
+        public String getUuid() {
+            return uuid;
+        }
+
         public String getName() {
             return name;
         }
@@ -411,12 +417,14 @@ public class LoadBalancerTO {
     public static class ConditionTO implements Serializable {
         private static final long serialVersionUID = 2L;
         private final Long id;
+        private final String uuid;
         private final long threshold;
         private final Condition.Operator relationalOperator;
         private final CounterTO counter;
 
-        public ConditionTO(Long id, long threshold, Condition.Operator relationalOperator, CounterTO counter) {
+        public ConditionTO(Long id, String uuid, long threshold, Condition.Operator relationalOperator, CounterTO counter) {
             this.id = id;
+            this.uuid = uuid;
             this.threshold = threshold;
             this.relationalOperator = relationalOperator;
             this.counter = counter;
@@ -426,6 +434,10 @@ public class LoadBalancerTO {
             return id;
         }
 
+        public String getUuid() {
+            return uuid;
+        }
+
         public long getThreshold() {
             return threshold;
         }
@@ -442,6 +454,7 @@ public class LoadBalancerTO {
     public static class AutoScalePolicyTO implements Serializable {
         private static final long serialVersionUID = 2L;
         private final long id;
+        private final String uuid;
         private final int duration;
         private final int quietTime;
         private final Date lastQuietTime;
@@ -449,8 +462,9 @@ public class LoadBalancerTO {
         boolean revoked;
         private final List<ConditionTO> conditions;
 
-        public AutoScalePolicyTO(long id, int duration, int quietTime, Date lastQuietTime, AutoScalePolicy.Action action, List<ConditionTO> conditions, boolean revoked) {
+        public AutoScalePolicyTO(long id, String uuid, int duration, int quietTime, Date lastQuietTime, AutoScalePolicy.Action action, List<ConditionTO> conditions, boolean revoked) {
             this.id = id;
+            this.uuid = uuid;
             this.duration = duration;
             this.quietTime = quietTime;
             this.lastQuietTime = lastQuietTime;
@@ -463,6 +477,10 @@ public class LoadBalancerTO {
             return id;
         }
 
+        public String getUuid() {
+            return uuid;
+        }
+
         public int getDuration() {
             return duration;
         }

api/src/main/java/com/cloud/agent/api/to/NetworkTO.java

@@ -36,7 +36,7 @@ public class NetworkTO {
     protected TrafficType type;
     protected URI broadcastUri;
     protected URI isolationUri;
-    protected boolean isSecurityGroupEnabled;
+    protected boolean securityGroupEnabled;
     protected String name;
     protected String ip6address;
     protected String ip6gateway;
@@ -112,7 +112,7 @@ public class NetworkTO {
     }
 
     public void setSecurityGroupEnabled(boolean enabled) {
-        this.isSecurityGroupEnabled = enabled;
+        this.securityGroupEnabled = enabled;
     }
 
     /**
@@ -221,7 +221,7 @@ public class NetworkTO {
     }
 
     public boolean isSecurityGroupEnabled() {
-        return this.isSecurityGroupEnabled;
+        return this.securityGroupEnabled;
     }
 
     public void setIp6Dns1(String ip6Dns1) {

api/src/main/java/com/cloud/agent/api/to/NfsTO.java

@@ -17,6 +17,7 @@
 package com.cloud.agent.api.to;
 
 import com.cloud.storage.DataStoreRole;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class NfsTO implements DataStoreTO {
 
@@ -41,6 +42,13 @@ public class NfsTO implements DataStoreTO {
 
     }
 
+    @Override
+    public String toString() {
+        return String.format("NfsTO %s",
+                ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                        this, "uuid", "_url", "_role", "nfsVersion"));
+    }
+
     @Override
     public String getUrl() {
         return _url;

api/src/main/java/com/cloud/agent/api/to/NicTO.java

@@ -86,6 +86,14 @@ public class NicTO extends NetworkTO {
         this.nicUuid = uuid;
     }
 
+    public String getNicUuid() {
+        return nicUuid;
+    }
+
+    public void setNicUuid(String nicUuid) {
+        this.nicUuid = nicUuid;
+    }
+
     @Override
     public String toString() {
         return new StringBuilder("[Nic:").append(type).append("-").append(ip).append("-").append(broadcastUri).append("]").toString();

api/src/main/java/com/cloud/agent/api/to/PortForwardingRuleTO.java

@@ -19,6 +19,7 @@ package com.cloud.agent.api.to;
 import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.PortForwardingRule;
 import com.cloud.utils.net.NetUtils;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * PortForwardingRuleTO specifies one port forwarding rule.
@@ -37,6 +38,7 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
         super(rule, srcVlanTag, srcIp);
         this.dstIp = rule.getDestinationIpAddress().addr();
         this.dstPortRange = new int[] {rule.getDestinationPortStart(), rule.getDestinationPortEnd()};
+        this.sourceCidrList = rule.getSourceCidrList();
     }
 
     public PortForwardingRuleTO(long id, String srcIp, int srcPortStart, int srcPortEnd, String dstIp, int dstPortStart, int dstPortEnd, String protocol,
@@ -58,4 +60,11 @@ public class PortForwardingRuleTO extends FirewallRuleTO {
         return NetUtils.portRangeToString(dstPortRange);
     }
 
+    public String getSourceCidrListAsString() {
+        if (sourceCidrList != null) {
+            return StringUtils.join(sourceCidrList, ",");
+        }
+        return null;
+    }
+
 }

api/src/main/java/com/cloud/agent/api/to/RemoteInstanceTO.java

@@ -18,40 +18,41 @@
  */
 package com.cloud.agent.api.to;
 
+import java.io.Serializable;
+
 import com.cloud.agent.api.LogLevel;
 import com.cloud.hypervisor.Hypervisor;
 
-import java.io.Serializable;
-
 public class RemoteInstanceTO implements Serializable {
 
     private Hypervisor.HypervisorType hypervisorType;
-    private String hostName;
     private String instanceName;
+    private String instancePath;
 
-    // Vmware Remote Instances parameters
+    // VMware Remote Instances parameters (required for exporting OVA through ovftool)
     // TODO: cloud.agent.transport.Request#getCommands() cannot handle gsoc decode for polymorphic classes
     private String vcenterUsername;
     @LogLevel(LogLevel.Log4jLevel.Off)
     private String vcenterPassword;
     private String vcenterHost;
     private String datacenterName;
-    private String clusterName;
 
     public RemoteInstanceTO() {
     }
 
-    public RemoteInstanceTO(String hostName, String instanceName, String vcenterHost,
-                            String datacenterName, String clusterName,
-                            String vcenterUsername, String vcenterPassword) {
+    public RemoteInstanceTO(String instanceName) {
         this.hypervisorType = Hypervisor.HypervisorType.VMware;
-        this.hostName = hostName;
         this.instanceName = instanceName;
+    }
+
+    public RemoteInstanceTO(String instanceName, String instancePath, String vcenterHost, String vcenterUsername, String vcenterPassword, String datacenterName) {
+        this.hypervisorType = Hypervisor.HypervisorType.VMware;
+        this.instanceName = instanceName;
+        this.instancePath = instancePath;
         this.vcenterHost = vcenterHost;
-        this.datacenterName = datacenterName;
-        this.clusterName = clusterName;
         this.vcenterUsername = vcenterUsername;
         this.vcenterPassword = vcenterPassword;
+        this.datacenterName = datacenterName;
     }
 
     public Hypervisor.HypervisorType getHypervisorType() {
@@ -62,8 +63,8 @@ public class RemoteInstanceTO implements Serializable {
         return this.instanceName;
     }
 
-    public String getHostName() {
-        return this.hostName;
+    public String getInstancePath() {
+        return this.instancePath;
     }
 
     public String getVcenterUsername() {
@@ -81,8 +82,4 @@ public class RemoteInstanceTO implements Serializable {
     public String getDatacenterName() {
         return datacenterName;
     }
-
-    public String getClusterName() {
-        return clusterName;
-    }
 }

api/src/main/java/com/cloud/agent/api/to/S3TO.java

@@ -22,6 +22,7 @@ import com.cloud.agent.api.LogLevel;
 import com.cloud.agent.api.LogLevel.Log4jLevel;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.utils.storage.S3.ClientOptions;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public final class S3TO implements ClientOptions, DataStoreTO {
 
@@ -68,6 +69,13 @@ public final class S3TO implements ClientOptions, DataStoreTO {
 
     }
 
+    @Override
+    public String toString() {
+        return String.format("S3TO %s",
+                ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                        this, "id", "uuid", "bucketName"));
+    }
+
     public Long getId() {
         return this.id;
     }

api/src/main/java/com/cloud/agent/api/to/StorageFilerTO.java

@@ -19,6 +19,7 @@ package com.cloud.agent.api.to;
 import com.cloud.agent.api.LogLevel;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StoragePool;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class StorageFilerTO {
     long id;
@@ -73,6 +74,6 @@ public class StorageFilerTO {
 
     @Override
     public String toString() {
-        return new StringBuilder("Pool[").append(id).append("|").append(host).append(":").append(port).append("|").append(path).append("]").toString();
+        return String.format("Pool %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "host", "port", "path"));
     }
 }

api/src/main/java/com/cloud/agent/api/to/SwiftTO.java

@@ -18,6 +18,7 @@ package com.cloud.agent.api.to;
 
 import com.cloud.storage.DataStoreRole;
 import com.cloud.utils.SwiftUtil;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
     Long id;
@@ -41,6 +42,13 @@ public class SwiftTO implements DataStoreTO, SwiftUtil.SwiftClientCfg {
         this.storagePolicy = storagePolicy;
     }
 
+    @Override
+    public String toString() {
+        return String.format("SwiftTO %s",
+                ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                        this, "id", "account", "userName"));
+    }
+
     public Long getId() {
         return id;
     }

api/src/main/java/com/cloud/agent/api/to/VirtualMachineTO.java

@@ -19,20 +19,22 @@ package com.cloud.agent.api.to;
 import java.util.List;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.stream.Collectors;
 
 import com.cloud.agent.api.LogLevel;
 import com.cloud.network.element.NetworkElement;
 import com.cloud.template.VirtualMachineTemplate.BootloaderType;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.VirtualMachine.Type;
+import com.cloud.vm.VmDetailConstants;
 
 public class VirtualMachineTO {
     private long id;
     private String name;
     private BootloaderType bootloader;
     private VirtualMachine.State state;
-    Type type;
-    int cpus;
+    private Type type;
+    private int cpus;
 
     /**
         'speed' is still here since 4.0.X/4.1.X management servers do not support
@@ -43,47 +45,50 @@ public class VirtualMachineTO {
          So this is here for backwards compatibility with 4.0.X/4.1.X management servers
          and newer agents.
     */
-    Integer speed;
-    Integer minSpeed;
-    Integer maxSpeed;
+    private Integer speed;
+    private Integer minSpeed;
+    private Integer maxSpeed;
 
-    long minRam;
-    long maxRam;
-    String hostName;
-    String arch;
-    String os;
-    String platformEmulator;
-    String bootArgs;
-    String[] bootupScripts;
-    boolean enableHA;
-    boolean limitCpuUse;
-    boolean enableDynamicallyScaleVm;
+    private long minRam;
+    private long maxRam;
+    private String hostName;
+    private String arch;
+    private String os;
+    private String platformEmulator;
+    private String bootArgs;
+    private String[] bootupScripts;
+    private boolean enableHA;
+    private boolean limitCpuUse;
+    private boolean enableDynamicallyScaleVm;
     @LogLevel(LogLevel.Log4jLevel.Off)
-    String vncPassword;
-    String vncAddr;
-    Map<String, String> params;
-    String uuid;
-    String bootType;
-    String bootMode;
-    boolean enterHardwareSetup;
+    private String vncPassword;
+    private String vncAddr;
+    private Map<String, String> details;
+    private Map<String, String> params;
+    private String uuid;
+    private String bootType;
+    private String bootMode;
+    private boolean enterHardwareSetup;
 
-    DiskTO[] disks;
-    NicTO[] nics;
-    GPUDeviceTO gpuDevice;
-    Integer vcpuMaxLimit;
-    List<String[]> vmData = null;
+    private DiskTO[] disks;
+    private NicTO[] nics;
+    private GPUDeviceTO gpuDevice;
+    private Integer vcpuMaxLimit;
+    private List<String[]> vmData = null;
 
-    String configDriveLabel = null;
-    String configDriveIsoRootFolder = null;
-    String configDriveIsoFile = null;
-    NetworkElement.Location configDriveLocation = NetworkElement.Location.SECONDARY;
+    private String configDriveLabel = null;
+    private String configDriveIsoRootFolder = null;
+    private String configDriveIsoFile = null;
+    private NetworkElement.Location configDriveLocation = NetworkElement.Location.SECONDARY;
 
-    Double cpuQuotaPercentage = null;
+    private Double cpuQuotaPercentage = null;
 
-    Map<String, String> guestOsDetails = new HashMap<String, String>();
-    Map<String, String> extraConfig = new HashMap<>();
-    Map<Long, String> networkIdToNetworkNameMap = new HashMap<>();
-    DeployAsIsInfoTO deployAsIsInfo;
+    private Map<String, String> guestOsDetails = new HashMap<String, String>();
+    private Map<String, String> extraConfig = new HashMap<>();
+    private Map<Long, String> networkIdToNetworkNameMap = new HashMap<>();
+    private DeployAsIsInfoTO deployAsIsInfo;
+    private String metadataManufacturer;
+    private String metadataProductName;
 
     public VirtualMachineTO(long id, String instanceName, VirtualMachine.Type type, int cpus, Integer speed, long minRam, long maxRam, BootloaderType bootloader,
             String os, boolean enableHA, boolean limitCpuUse, String vncPassword) {
@@ -189,7 +194,11 @@ public class VirtualMachineTO {
         return maxSpeed;
     }
 
-    public boolean getLimitCpuUse() {
+    public boolean isEnableHA() {
+        return enableHA;
+    }
+
+    public boolean isLimitCpuUse() {
         return limitCpuUse;
     }
 
@@ -254,6 +263,10 @@ public class VirtualMachineTO {
         this.bootupScripts = bootupScripts;
     }
 
+    public void setEnableHA(boolean enableHA) {
+        this.enableHA = enableHA;
+    }
+
     public DiskTO[] getDisks() {
         return disks;
     }
@@ -287,11 +300,11 @@ public class VirtualMachineTO {
     }
 
     public Map<String, String> getDetails() {
-        return params;
+        return details;
     }
 
     public void setDetails(Map<String, String> params) {
-        this.params = params;
+        this.details = params;
     }
 
     public String getUuid() {
@@ -429,8 +442,72 @@ public class VirtualMachineTO {
         this.deployAsIsInfo = deployAsIsInfo;
     }
 
+    public void setSpeed(Integer speed) {
+        this.speed = speed;
+    }
+
+    public void setMinSpeed(Integer minSpeed) {
+        this.minSpeed = minSpeed;
+    }
+
+    public void setMaxSpeed(Integer maxSpeed) {
+        this.maxSpeed = maxSpeed;
+    }
+
+    public void setMinRam(long minRam) {
+        this.minRam = minRam;
+    }
+
+    public void setMaxRam(long maxRam) {
+        this.maxRam = maxRam;
+    }
+
+    public void setLimitCpuUse(boolean limitCpuUse) {
+        this.limitCpuUse = limitCpuUse;
+    }
+
+    public Map<String, String> getParams() {
+        return params;
+    }
+
+    public void setParams(Map<String, String> params) {
+        this.params = params;
+    }
+
+    public void setExtraConfig(Map<String, String> extraConfig) {
+        this.extraConfig = extraConfig;
+    }
+
+    public String getMetadataManufacturer() {
+        return metadataManufacturer;
+    }
+
+    public void setMetadataManufacturer(String metadataManufacturer) {
+        this.metadataManufacturer = metadataManufacturer;
+    }
+
+    public String getMetadataProductName() {
+        return metadataProductName;
+    }
+
+    public void setMetadataProductName(String metadataProductName) {
+        this.metadataProductName = metadataProductName;
+    }
+
     @Override
     public String toString() {
         return String.format("VM {id: \"%s\", name: \"%s\", uuid: \"%s\", type: \"%s\"}", id, name, uuid, type);
     }
+
+    public Map<String, String> getExternalDetails() {
+        if (details == null) {
+            return new HashMap<>();
+        }
+        return details.entrySet().stream()
+                .filter(entry -> entry.getKey().startsWith(VmDetailConstants.EXTERNAL_DETAIL_PREFIX))
+                .collect(Collectors.toMap(
+                        entry -> entry.getKey().substring(VmDetailConstants.EXTERNAL_DETAIL_PREFIX.length()),
+                        Map.Entry::getValue
+                ));
+    }
 }

api/src/main/java/com/cloud/bgp/ASNumber.java

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.bgp;
+
+import org.apache.cloudstack.acl.InfrastructureEntity;
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+import java.util.Date;
+
+public interface ASNumber extends InfrastructureEntity, InternalIdentity, Identity {
+
+    Long getAccountId();
+    Long getDomainId();
+    long getAsNumber();
+    long getAsNumberRangeId();
+    long getDataCenterId();
+    Date getAllocatedTime();
+    boolean isAllocated();
+    Long getNetworkId();
+    Long getVpcId();
+    Date getCreated();
+    Date getRemoved();
+}

api/src/main/java/com/cloud/bgp/ASNumberRange.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.bgp;
+
+import org.apache.cloudstack.acl.InfrastructureEntity;
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+import java.util.Date;
+
+public interface ASNumberRange extends InfrastructureEntity, InternalIdentity, Identity {
+
+    long getStartASNumber();
+    long getEndASNumber();
+    long getDataCenterId();
+    Date getCreated();
+}

api/src/main/java/com/cloud/bgp/BGPService.java

@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.bgp;
+
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.Network;
+import com.cloud.network.vpc.Vpc;
+import com.cloud.utils.Pair;
+import org.apache.cloudstack.api.command.user.bgp.ListASNumbersCmd;
+import org.apache.cloudstack.network.BgpPeer;
+
+import java.util.List;
+
+public interface BGPService {
+
+    ASNumberRange createASNumberRange(long zoneId, long startASNumber, long endASNumber);
+    List<ASNumberRange> listASNumberRanges(Long zoneId);
+    Pair<List<ASNumber>, Integer> listASNumbers(ListASNumbersCmd cmd);
+    boolean allocateASNumber(long zoneId, Long asNumber, Long networkId, Long vpcId);
+    Pair<Boolean, String> releaseASNumber(long zoneId, long asNumber, boolean isReleaseNetworkDestroy);
+    boolean deleteASRange(long id);
+
+    boolean applyBgpPeers(Network network, boolean continueOnError) throws ResourceUnavailableException;
+
+    boolean applyBgpPeers(Vpc vpc, boolean continueOnError) throws ResourceUnavailableException;
+
+    List<? extends BgpPeer> getBgpPeersForNetwork(Network network);
+
+    List<? extends BgpPeer> getBgpPeersForVpc(Vpc vpc);
+}

api/src/main/java/com/cloud/capacity/Capacity.java

@@ -34,13 +34,17 @@ public interface Capacity extends InternalIdentity, Identity {
     public static final short CAPACITY_TYPE_LOCAL_STORAGE = 9;
     public static final short CAPACITY_TYPE_VIRTUAL_NETWORK_IPV6_SUBNET = 10;
     public static final short CAPACITY_TYPE_GPU = 19;
+    public static final short CAPACITY_TYPE_OBJECT_STORAGE = 20;
+    public static final short CAPACITY_TYPE_BACKUP_STORAGE = 21;
 
     public static final short CAPACITY_TYPE_CPU_CORE = 90;
 
     public static final List<Short> STORAGE_CAPACITY_TYPES = List.of(CAPACITY_TYPE_STORAGE,
             CAPACITY_TYPE_STORAGE_ALLOCATED,
             CAPACITY_TYPE_SECONDARY_STORAGE,
-            CAPACITY_TYPE_LOCAL_STORAGE);
+            CAPACITY_TYPE_LOCAL_STORAGE,
+            CAPACITY_TYPE_BACKUP_STORAGE,
+            CAPACITY_TYPE_OBJECT_STORAGE);
 
     public Long getHostOrPoolId();
 

api/src/main/java/com/cloud/configuration/ConfigurationService.java

@@ -17,7 +17,11 @@
 package com.cloud.configuration;
 
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 
+import com.cloud.network.Network;
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.command.admin.config.ResetCfgCmd;
 import org.apache.cloudstack.api.command.admin.config.UpdateCfgCmd;
 import org.apache.cloudstack.api.command.admin.network.CreateGuestNetworkIpv6PrefixCmd;
@@ -201,11 +205,12 @@ public interface ConfigurationService {
      *            TODO
      * @param allocationState
      *            TODO
+     * @param storageAccessGroups
      * @return the new pod if successful, null otherwise
      * @throws
      * @throws
      */
-    Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState);
+    Pod createPod(long zoneId, String name, String startIp, String endIp, String gateway, String netmask, String allocationState, List<String> storageAccessGroups);
 
     /**
      * Creates a mutual exclusive IP range in the pod with same gateway, netmask.
@@ -372,4 +377,16 @@ public interface ConfigurationService {
     List<? extends PortableIp> listPortableIps(long id);
 
     Boolean isAccountAllowedToCreateOfferingsWithTags(IsAccountAllowedToCreateOfferingsWithTagsCmd cmd);
+
+    public static final Map<String, String> ProviderDetailKeyMap = Map.of(
+            Network.Provider.Nsx.getName(), ApiConstants.NSX_DETAIL_KEY,
+            Network.Provider.Netris.getName(), ApiConstants.NETRIS_DETAIL_KEY
+    );
+
+    public static boolean IsIpRangeForProvider(Network.Provider provider) {
+        if (Objects.isNull(provider)) {
+            return false;
+        }
+        return ProviderDetailKeyMap.containsKey(provider.getName());
+    }
 }

api/src/main/java/com/cloud/configuration/Resource.java

@@ -21,7 +21,7 @@ public interface Resource {
     short RESOURCE_UNLIMITED = -1;
     String UNLIMITED = "Unlimited";
 
-    enum ResourceType { // Primary and Secondary storage are allocated_storage and not the physical storage.
+    enum ResourceType { // All storage type resources are allocated_storage and not the physical storage.
         user_vm("user_vm", 0),
         public_ip("public_ip", 1),
         volume("volume", 2),
@@ -33,7 +33,12 @@ public interface Resource {
         cpu("cpu", 8),
         memory("memory", 9),
         primary_storage("primary_storage", 10),
-        secondary_storage("secondary_storage", 11);
+        secondary_storage("secondary_storage", 11),
+        backup("backup", 12),
+        backup_storage("backup_storage", 13),
+        bucket("bucket", 14),
+        object_storage("object_storage", 15),
+        gpu("gpu", 16);
 
         private String name;
         private int ordinal;
@@ -62,6 +67,10 @@ public interface Resource {
             }
             return null;
         }
+
+        public static Boolean isStorageType(ResourceType type) {
+            return (type == primary_storage || type == secondary_storage || type == backup_storage || type == object_storage);
+        }
     }
 
     public static class ResourceOwnerType {

api/src/main/java/com/cloud/cpu/CPU.java

@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.cpu;
+
+import org.apache.commons.lang3.StringUtils;
+
+public class CPU {
+    public enum CPUArch {
+        x86("i686", 32),
+        amd64("x86_64", 64),
+        arm64("aarch64", 64);
+
+        private final String type;
+        private final int bits;
+
+        CPUArch(String type, int bits) {
+            this.type = type;
+            this.bits = bits;
+        }
+
+        public static CPUArch getDefault() {
+            return amd64;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        public int getBits() {
+            return bits;
+        }
+
+        public static CPUArch fromType(String type) {
+            if (StringUtils.isBlank(type)) {
+                return getDefault();
+            }
+            for (CPUArch arch : values()) {
+                if (arch.type.equals(type)) {
+                    return arch;
+                }
+            }
+            throw new IllegalArgumentException("Unsupported arch type: " + type);
+        }
+
+        public static String getTypesAsCSV() {
+            StringBuilder sb = new StringBuilder();
+            for (CPUArch arch : values()) {
+                sb.append(arch.getType()).append(",");
+            }
+            if (sb.length() > 0) {
+                sb.setLength(sb.length() - 1);
+            }
+            return sb.toString();
+        }
+    }
+}

api/src/main/java/com/cloud/dc/DedicatedResources.java

@@ -21,6 +21,10 @@ import org.apache.cloudstack.api.Identity;
 import org.apache.cloudstack.api.InternalIdentity;
 
 public interface DedicatedResources extends InfrastructureEntity, InternalIdentity, Identity {
+    enum Type {
+        Zone, Pod, Cluster, Host
+    }
+
     @Override
     long getId();
 

api/src/main/java/com/cloud/dc/Pod.java

@@ -43,4 +43,6 @@ public interface Pod extends InfrastructureEntity, Grouping, Identity, InternalI
     AllocationState getAllocationState();
 
     boolean getExternalDhcp();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java

@@ -62,7 +62,7 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner {
             "vm.allocation.algorithm",
             "Advanced",
             "random",
-            "Order in which hosts within a cluster will be considered for VM/volume allocation. The value can be 'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', or 'firstfitleastconsumed'.",
+            "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', 'firstfit', 'userdispersing', 'userconcentratedpod_random', 'userconcentratedpod_firstfit', or 'firstfitleastconsumed'.",
             true,
             ConfigKey.Scope.Global, null, null, null, null, null,
             ConfigKey.Kind.Select,

api/src/main/java/com/cloud/event/EventTypes.java

@@ -28,10 +28,19 @@ import org.apache.cloudstack.api.response.HostResponse;
 import org.apache.cloudstack.api.response.PodResponse;
 import org.apache.cloudstack.api.response.ZoneResponse;
 import org.apache.cloudstack.config.Configuration;
+import org.apache.cloudstack.datacenter.DataCenterIpv4GuestSubnet;
+import org.apache.cloudstack.extension.Extension;
+import org.apache.cloudstack.extension.ExtensionCustomAction;
+import org.apache.cloudstack.gpu.GpuCard;
+import org.apache.cloudstack.gpu.GpuDevice;
+import org.apache.cloudstack.gpu.VgpuProfile;
 import org.apache.cloudstack.ha.HAConfig;
+import org.apache.cloudstack.network.BgpPeer;
+import org.apache.cloudstack.network.Ipv4GuestSubnetNetworkMap;
+import org.apache.cloudstack.quota.QuotaTariff;
 import org.apache.cloudstack.storage.object.Bucket;
 import org.apache.cloudstack.storage.object.ObjectStore;
-import org.apache.cloudstack.quota.QuotaTariff;
+import org.apache.cloudstack.storage.sharedfs.SharedFS;
 import org.apache.cloudstack.usage.Usage;
 import org.apache.cloudstack.vm.schedule.VMSchedule;
 
@@ -242,6 +251,8 @@ public class EventTypes {
     public static final String EVENT_ROLE_UPDATE = "ROLE.UPDATE";
     public static final String EVENT_ROLE_DELETE = "ROLE.DELETE";
     public static final String EVENT_ROLE_IMPORT = "ROLE.IMPORT";
+    public static final String EVENT_ROLE_ENABLE = "ROLE.ENABLE";
+    public static final String EVENT_ROLE_DISABLE = "ROLE.DISABLE";
     public static final String EVENT_ROLE_PERMISSION_CREATE = "ROLE.PERMISSION.CREATE";
     public static final String EVENT_ROLE_PERMISSION_UPDATE = "ROLE.PERMISSION.UPDATE";
     public static final String EVENT_ROLE_PERMISSION_DELETE = "ROLE.PERMISSION.DELETE";
@@ -283,9 +294,12 @@ public class EventTypes {
 
     //registering userdata events
     public static final String EVENT_REGISTER_USER_DATA = "REGISTER.USER.DATA";
+    public static final String EVENT_REGISTER_CNI_CONFIG = "REGISTER.CNI.CONFIG";
+    public static final String EVENT_DELETE_CNI_CONFIG = "DELETE.CNI.CONFIG";
 
     //register for user API and secret keys
     public static final String EVENT_REGISTER_FOR_SECRET_API_KEY = "REGISTER.USER.KEY";
+    public static final String API_KEY_ACCESS_UPDATE = "API.KEY.ACCESS.UPDATE";
 
     // Template Events
     public static final String EVENT_TEMPLATE_CREATE = "TEMPLATE.CREATE";
@@ -316,6 +330,8 @@ public class EventTypes {
     public static final String EVENT_VOLUME_UPDATE = "VOLUME.UPDATE";
     public static final String EVENT_VOLUME_DESTROY = "VOLUME.DESTROY";
     public static final String EVENT_VOLUME_RECOVER = "VOLUME.RECOVER";
+    public static final String EVENT_VOLUME_IMPORT = "VOLUME.IMPORT";
+    public static final String EVENT_VOLUME_UNMANAGE = "VOLUME.UNMANAGE";
     public static final String EVENT_VOLUME_CHANGE_DISK_OFFERING = "VOLUME.CHANGE.DISK.OFFERING";
 
     // Domains
@@ -331,6 +347,7 @@ public class EventTypes {
     public static final String EVENT_SNAPSHOT_OFF_PRIMARY = "SNAPSHOT.OFF_PRIMARY";
     public static final String EVENT_SNAPSHOT_DELETE = "SNAPSHOT.DELETE";
     public static final String EVENT_SNAPSHOT_REVERT = "SNAPSHOT.REVERT";
+    public static final String EVENT_SNAPSHOT_EXTRACT = "SNAPSHOT.EXTRACT";
     public static final String EVENT_SNAPSHOT_POLICY_CREATE = "SNAPSHOTPOLICY.CREATE";
     public static final String EVENT_SNAPSHOT_POLICY_UPDATE = "SNAPSHOTPOLICY.UPDATE";
     public static final String EVENT_SNAPSHOT_POLICY_DELETE = "SNAPSHOTPOLICY.DELETE";
@@ -364,6 +381,21 @@ public class EventTypes {
     public static final String EVENT_DISK_OFFERING_EDIT = "DISK.OFFERING.EDIT";
     public static final String EVENT_DISK_OFFERING_DELETE = "DISK.OFFERING.DELETE";
 
+    // GPU Cards
+    public static final String EVENT_GPU_CARD_CREATE = "GPU.CARD.CREATE";
+    public static final String EVENT_GPU_CARD_EDIT = "GPU.CARD.EDIT";
+    public static final String EVENT_GPU_CARD_DELETE = "GPU.CARD.DELETE";
+
+    // vGPU Profile
+    public static final String EVENT_VGPU_PROFILE_CREATE = "VGPU.PROFILE.CREATE";
+    public static final String EVENT_VGPU_PROFILE_EDIT = "VGPU.PROFILE.EDIT";
+    public static final String EVENT_VGPU_PROFILE_DELETE = "VGPU.PROFILE.DELETE";
+
+    // GPU Devices
+    public static final String EVENT_GPU_DEVICE_CREATE = "GPU.DEVICE.CREATE";
+    public static final String EVENT_GPU_DEVICE_EDIT = "GPU.DEVICE.EDIT";
+    public static final String EVENT_GPU_DEVICE_DELETE = "GPU.DEVICE.DELETE";
+
     // Network offerings
     public static final String EVENT_NETWORK_OFFERING_CREATE = "NETWORK.OFFERING.CREATE";
     public static final String EVENT_NETWORK_OFFERING_ASSIGN = "NETWORK.OFFERING.ASSIGN";
@@ -388,6 +420,11 @@ public class EventTypes {
     public static final String EVENT_VLAN_IP_RANGE_RELEASE = "VLAN.IP.RANGE.RELEASE";
     public static final String EVENT_VLAN_IP_RANGE_UPDATE = "VLAN.IP.RANGE.UPDATE";
 
+    // AS Number
+    public static final String EVENT_AS_RANGE_CREATE = "AS.RANGE.CREATE";
+    public static final String EVENT_AS_RANGE_DELETE = "AS.RANGE.DELETE";
+    public static final String EVENT_AS_NUMBER_RELEASE = "AS.NUMBER.RELEASE";
+
     public static final String EVENT_MANAGEMENT_IP_RANGE_CREATE = "MANAGEMENT.IP.RANGE.CREATE";
     public static final String EVENT_MANAGEMENT_IP_RANGE_DELETE = "MANAGEMENT.IP.RANGE.DELETE";
     public static final String EVENT_MANAGEMENT_IP_RANGE_UPDATE = "MANAGEMENT.IP.RANGE.UPDATE";
@@ -446,9 +483,12 @@ public class EventTypes {
     public static final String EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE = "MAINT.PREPARE.PS";
 
     // Primary storage pool
+    public static final String EVENT_UPDATE_PRIMARY_STORAGE = "UPDATE.PS";
     public static final String EVENT_ENABLE_PRIMARY_STORAGE = "ENABLE.PS";
     public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS";
     public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL";
+    public static final String EVENT_CONFIGURE_STORAGE_ACCESS = "CONFIGURE.STORAGE.ACCESS";
+    public static final String EVENT_CHANGE_STORAGE_POOL_SCOPE = "CHANGE.STORAGE.POOL.SCOPE";
 
     // VPN
     public static final String EVENT_REMOTE_ACCESS_VPN_CREATE = "VPN.REMOTE.ACCESS.CREATE";
@@ -479,6 +519,8 @@ public class EventTypes {
 
     public static final String EVENT_ZONE_VLAN_ASSIGN = "ZONE.VLAN.ASSIGN";
     public static final String EVENT_ZONE_VLAN_RELEASE = "ZONE.VLAN.RELEASE";
+    public static final String EVENT_ZONE_VXLAN_ASSIGN = "ZONE.VXLAN.ASSIGN";
+    public static final String EVENT_ZONE_VXLAN_RELEASE = "ZONE.VXLAN.RELEASE";
 
     // Projects
     public static final String EVENT_PROJECT_CREATE = "PROJECT.CREATE";
@@ -590,11 +632,13 @@ public class EventTypes {
     public static final String EVENT_VM_BACKUP_CREATE = "BACKUP.CREATE";
     public static final String EVENT_VM_BACKUP_RESTORE = "BACKUP.RESTORE";
     public static final String EVENT_VM_BACKUP_DELETE = "BACKUP.DELETE";
+    public static final String EVENT_VM_BACKUP_OFFERING_REMOVED_AND_BACKUPS_DELETED = "BACKUP.OFFERING.BACKUPS.DEL";
     public static final String EVENT_VM_BACKUP_RESTORE_VOLUME_TO_VM = "BACKUP.RESTORE.VOLUME.TO.VM";
     public static final String EVENT_VM_BACKUP_SCHEDULE_CONFIGURE = "BACKUP.SCHEDULE.CONFIGURE";
     public static final String EVENT_VM_BACKUP_SCHEDULE_DELETE = "BACKUP.SCHEDULE.DELETE";
     public static final String EVENT_VM_BACKUP_USAGE_METRIC = "BACKUP.USAGE.METRIC";
     public static final String EVENT_VM_BACKUP_EDIT = "BACKUP.OFFERING.EDIT";
+    public static final String EVENT_VM_CREATE_FROM_BACKUP = "VM.CREATE.FROM.BACKUP";
 
     // external network device events
     public static final String EVENT_EXTERNAL_NVP_CONTROLLER_ADD = "PHYSICAL.NVPCONTROLLER.ADD";
@@ -670,6 +714,9 @@ public class EventTypes {
     public static final String EVENT_EXTERNAL_OPENDAYLIGHT_CONFIGURE_CONTROLLER = "PHYSICAL.ODLCONTROLLER.CONFIGURE";
 
     //Guest OS related events
+    public static final String EVENT_GUEST_OS_CATEGORY_ADD = "GUEST.OS.CATEGORY.ADD";
+    public static final String EVENT_GUEST_OS_CATEGORY_DELETE = "GUEST.OS.CATEGORY.DELETE";
+    public static final String EVENT_GUEST_OS_CATEGORY_UPDATE = "GUEST.OS.CATEGORY.UPDATE";
     public static final String EVENT_GUEST_OS_ADD = "GUEST.OS.ADD";
     public static final String EVENT_GUEST_OS_REMOVE = "GUEST.OS.REMOVE";
     public static final String EVENT_GUEST_OS_UPDATE = "GUEST.OS.UPDATE";
@@ -719,6 +766,15 @@ public class EventTypes {
 
     // SystemVM
     public static final String EVENT_LIVE_PATCH_SYSTEMVM = "LIVE.PATCH.SYSTEM.VM";
+    //Purge resources
+    public static final String EVENT_PURGE_EXPUNGED_RESOURCES = "PURGE.EXPUNGED.RESOURCES";
+
+    // Management Server
+    public static final String EVENT_MS_MAINTENANCE_PREPARE = "MS.MAINTENANCE.PREPARE";
+    public static final String EVENT_MS_MAINTENANCE_CANCEL = "MS.MAINTENANCE.CANCEL";
+    public static final String EVENT_MS_SHUTDOWN_PREPARE = "MS.SHUTDOWN.PREPARE";
+    public static final String EVENT_MS_SHUTDOWN_CANCEL = "MS.SHUTDOWN.CANCEL";
+    public static final String EVENT_MS_SHUTDOWN = "MS.SHUTDOWN";
 
     // OBJECT STORE
     public static final String EVENT_OBJECT_STORE_CREATE = "OBJECT.STORE.CREATE";
@@ -735,6 +791,67 @@ public class EventTypes {
     public static final String EVENT_QUOTA_TARIFF_DELETE = "QUOTA.TARIFF.DELETE";
     public static final String EVENT_QUOTA_TARIFF_UPDATE = "QUOTA.TARIFF.UPDATE";
 
+    // Routing
+    public static final String EVENT_ZONE_IP4_SUBNET_CREATE = "ZONE.IP4.SUBNET.CREATE";
+    public static final String EVENT_ZONE_IP4_SUBNET_UPDATE = "ZONE.IP4.SUBNET.UPDATE";
+    public static final String EVENT_ZONE_IP4_SUBNET_DELETE = "ZONE.IP4.SUBNET.DELETE";
+    public static final String EVENT_ZONE_IP4_SUBNET_DEDICATE = "ZONE.IP4.SUBNET.DEDICATE";
+    public static final String EVENT_ZONE_IP4_SUBNET_RELEASE = "ZONE.IP4.SUBNET.RELEASE";
+    public static final String EVENT_IP4_GUEST_SUBNET_CREATE = "IP4.GUEST.SUBNET.CREATE";
+    public static final String EVENT_IP4_GUEST_SUBNET_DELETE = "IP4.GUEST.SUBNET.DELETE";
+    public static final String EVENT_ROUTING_IPV4_FIREWALL_RULE_CREATE = "ROUTING.IPV4.FIREWALL.RULE.CREATE";
+    public static final String EVENT_ROUTING_IPV4_FIREWALL_RULE_UPDATE = "ROUTING.IPV4.FIREWALL.RULE.UPDATE";
+    public static final String EVENT_ROUTING_IPV4_FIREWALL_RULE_DELETE = "ROUTING.IPV4.FIREWALL.RULE.DELETE";
+    public static final String EVENT_BGP_PEER_CREATE = "BGP.PEER.CREATE";
+    public static final String EVENT_BGP_PEER_UPDATE = "BGP.PEER.UPDATE";
+    public static final String EVENT_BGP_PEER_DELETE = "BGP.PEER.DELETE";
+    public static final String EVENT_BGP_PEER_DEDICATE = "BGP.PEER.DEDICATE";
+    public static final String EVENT_BGP_PEER_RELEASE = "BGP.PEER.RELEASE";
+    public static final String EVENT_NETWORK_BGP_PEER_UPDATE = "NETWORK.BGP.PEER.UPDATE";
+    public static final String EVENT_VPC_BGP_PEER_UPDATE = "VPC.BGP.PEER.UPDATE";
+
+    // SharedFS
+    public static final String EVENT_SHAREDFS_CREATE = "SHAREDFS.CREATE";
+    public static final String EVENT_SHAREDFS_START = "SHAREDFS.START";
+    public static final String EVENT_SHAREDFS_UPDATE = "SHAREDFS.UPDATE";
+    public static final String EVENT_SHAREDFS_CHANGE_SERVICE_OFFERING = "SHAREDFS.CHANGE.SERVICE.OFFERING";
+    public static final String EVENT_SHAREDFS_CHANGE_DISK_OFFERING = "SHAREDFS.CHANGE.DISK.OFFERING";
+    public static final String EVENT_SHAREDFS_STOP = "SHAREDFS.STOP";
+    public static final String EVENT_SHAREDFS_RESTART = "SHAREDFS.RESTART";
+    public static final String EVENT_SHAREDFS_DESTROY = "SHAREDFS.DESTROY";
+    public static final String EVENT_SHAREDFS_EXPUNGE = "SHAREDFS.EXPUNGE";
+    public static final String EVENT_SHAREDFS_RECOVER = "SHAREDFS.RECOVER";
+
+    // Resource Limit
+    public static final String EVENT_RESOURCE_LIMIT_UPDATE = "RESOURCE.LIMIT.UPDATE";
+
+    // Management Server
+    public static final String EVENT_MANAGEMENT_SERVER_REMOVE = "MANAGEMENT.SERVER.REMOVE";
+
+    // VM Lease
+    public static final String VM_LEASE_EXPIRED = "VM.LEASE.EXPIRED";
+    public static final String VM_LEASE_DISABLED = "VM.LEASE.DISABLED";
+    public static final String VM_LEASE_CANCELLED = "VM.LEASE.CANCELLED";
+    public static final String VM_LEASE_EXPIRING = "VM.LEASE.EXPIRING";
+
+    // GUI Theme
+    public static final String EVENT_GUI_THEME_CREATE = "GUI.THEME.CREATE";
+    public static final String EVENT_GUI_THEME_REMOVE = "GUI.THEME.REMOVE";
+    public static final String EVENT_GUI_THEME_UPDATE = "GUI.THEME.UPDATE";
+
+    // Extension
+    public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE";
+    public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE";
+    public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE";
+    public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER";
+    public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_UPDATE = "EXTENSION.CUSTOM.ACTION.UPDATE";
+    public static final String EVENT_EXTENSION_CUSTOM_ACTION_DELETE = "EXTENSION.CUSTOM.ACTION.DELETE";
+
+    // Custom Action
+    public static final String EVENT_CUSTOM_ACTION = "CUSTOM.ACTION";
+
     static {
 
         // TODO: need a way to force author adding event types to declare the entity details as well, with out braking
@@ -836,6 +953,8 @@ public class EventTypes {
         entityEventDetails.put(EVENT_ROLE_UPDATE, Role.class);
         entityEventDetails.put(EVENT_ROLE_DELETE, Role.class);
         entityEventDetails.put(EVENT_ROLE_IMPORT, Role.class);
+        entityEventDetails.put(EVENT_ROLE_ENABLE, Role.class);
+        entityEventDetails.put(EVENT_ROLE_DISABLE, Role.class);
         entityEventDetails.put(EVENT_ROLE_PERMISSION_CREATE, RolePermission.class);
         entityEventDetails.put(EVENT_ROLE_PERMISSION_UPDATE, RolePermission.class);
         entityEventDetails.put(EVENT_ROLE_PERMISSION_DELETE, RolePermission.class);
@@ -892,6 +1011,7 @@ public class EventTypes {
         // Snapshots
         entityEventDetails.put(EVENT_SNAPSHOT_CREATE, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_DELETE, Snapshot.class);
+        entityEventDetails.put(EVENT_SNAPSHOT_EXTRACT, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_ON_PRIMARY, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_OFF_PRIMARY, Snapshot.class);
         entityEventDetails.put(EVENT_SNAPSHOT_POLICY_CREATE, SnapshotPolicy.class);
@@ -926,6 +1046,21 @@ public class EventTypes {
         entityEventDetails.put(EVENT_DISK_OFFERING_EDIT, DiskOffering.class);
         entityEventDetails.put(EVENT_DISK_OFFERING_DELETE, DiskOffering.class);
 
+        // GPU Cards
+        entityEventDetails.put(EVENT_GPU_CARD_CREATE, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_EDIT, GpuCard.class);
+        entityEventDetails.put(EVENT_GPU_CARD_DELETE, GpuCard.class);
+
+        // vGPU Profiles
+        entityEventDetails.put(EVENT_VGPU_PROFILE_CREATE, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_EDIT, VgpuProfile.class);
+        entityEventDetails.put(EVENT_VGPU_PROFILE_DELETE, VgpuProfile.class);
+
+        // GPU Devices
+        entityEventDetails.put(EVENT_GPU_DEVICE_CREATE, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_EDIT, GpuDevice.class);
+        entityEventDetails.put(EVENT_GPU_DEVICE_DELETE, GpuDevice.class);
+
         // Network offerings
         entityEventDetails.put(EVENT_NETWORK_OFFERING_CREATE, NetworkOffering.class);
         entityEventDetails.put(EVENT_NETWORK_OFFERING_ASSIGN, NetworkOffering.class);
@@ -996,8 +1131,10 @@ public class EventTypes {
         entityEventDetails.put(EVENT_MAINTENANCE_PREPARE_PRIMARY_STORAGE, Host.class);
 
         // Primary storage pool
+        entityEventDetails.put(EVENT_UPDATE_PRIMARY_STORAGE, StoragePool.class);
         entityEventDetails.put(EVENT_ENABLE_PRIMARY_STORAGE, StoragePool.class);
         entityEventDetails.put(EVENT_DISABLE_PRIMARY_STORAGE, StoragePool.class);
+        entityEventDetails.put(EVENT_CHANGE_STORAGE_POOL_SCOPE, StoragePool.class);
 
         // VPN
         entityEventDetails.put(EVENT_REMOTE_ACCESS_VPN_CREATE, RemoteAccessVpn.class);
@@ -1175,6 +1312,12 @@ public class EventTypes {
         entityEventDetails.put(EVENT_UPDATE_IMAGE_STORE_ACCESS_STATE, ImageStore.class);
         entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs");
 
+        entityEventDetails.put(EVENT_MS_MAINTENANCE_PREPARE, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_MAINTENANCE_CANCEL, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_SHUTDOWN_PREPARE, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_SHUTDOWN_CANCEL, "ManagementServer");
+        entityEventDetails.put(EVENT_MS_SHUTDOWN, "ManagementServer");
+
         //Object Store
         entityEventDetails.put(EVENT_OBJECT_STORE_CREATE, ObjectStore.class);
         entityEventDetails.put(EVENT_OBJECT_STORE_UPDATE, ObjectStore.class);
@@ -1189,8 +1332,65 @@ public class EventTypes {
         entityEventDetails.put(EVENT_QUOTA_TARIFF_CREATE, QuotaTariff.class);
         entityEventDetails.put(EVENT_QUOTA_TARIFF_DELETE, QuotaTariff.class);
         entityEventDetails.put(EVENT_QUOTA_TARIFF_UPDATE, QuotaTariff.class);
+
+        // Routing
+        entityEventDetails.put(EVENT_ZONE_IP4_SUBNET_CREATE, DataCenterIpv4GuestSubnet.class);
+        entityEventDetails.put(EVENT_ZONE_IP4_SUBNET_UPDATE, DataCenterIpv4GuestSubnet.class);
+        entityEventDetails.put(EVENT_ZONE_IP4_SUBNET_DELETE, DataCenterIpv4GuestSubnet.class);
+        entityEventDetails.put(EVENT_ZONE_IP4_SUBNET_DEDICATE, DataCenterIpv4GuestSubnet.class);
+        entityEventDetails.put(EVENT_ZONE_IP4_SUBNET_RELEASE, DataCenterIpv4GuestSubnet.class);
+        entityEventDetails.put(EVENT_IP4_GUEST_SUBNET_CREATE, Ipv4GuestSubnetNetworkMap.class);
+        entityEventDetails.put(EVENT_IP4_GUEST_SUBNET_DELETE, Ipv4GuestSubnetNetworkMap.class);
+        entityEventDetails.put(EVENT_ROUTING_IPV4_FIREWALL_RULE_CREATE, FirewallRule.class);
+        entityEventDetails.put(EVENT_ROUTING_IPV4_FIREWALL_RULE_UPDATE, FirewallRule.class);
+        entityEventDetails.put(EVENT_ROUTING_IPV4_FIREWALL_RULE_DELETE, FirewallRule.class);
+        entityEventDetails.put(EVENT_BGP_PEER_CREATE, BgpPeer.class);
+        entityEventDetails.put(EVENT_BGP_PEER_UPDATE, BgpPeer.class);
+        entityEventDetails.put(EVENT_BGP_PEER_DELETE, BgpPeer.class);
+        entityEventDetails.put(EVENT_BGP_PEER_DEDICATE, BgpPeer.class);
+        entityEventDetails.put(EVENT_BGP_PEER_RELEASE, BgpPeer.class);
+
+        // SharedFS
+        entityEventDetails.put(EVENT_SHAREDFS_CREATE, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_START, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_STOP, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_UPDATE, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_CHANGE_SERVICE_OFFERING, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_CHANGE_DISK_OFFERING, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_RESTART, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_DESTROY, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_EXPUNGE, SharedFS.class);
+        entityEventDetails.put(EVENT_SHAREDFS_RECOVER, SharedFS.class);
+
+        // Management Server
+        entityEventDetails.put(EVENT_MANAGEMENT_SERVER_REMOVE, "ManagementServer");
+
+        // VM Lease
+        entityEventDetails.put(VM_LEASE_EXPIRED, VirtualMachine.class);
+        entityEventDetails.put(VM_LEASE_EXPIRING, VirtualMachine.class);
+        entityEventDetails.put(VM_LEASE_DISABLED, VirtualMachine.class);
+        entityEventDetails.put(VM_LEASE_CANCELLED, VirtualMachine.class);
+
+        // GUI theme
+        entityEventDetails.put(EVENT_GUI_THEME_CREATE, "GuiTheme");
+        entityEventDetails.put(EVENT_GUI_THEME_REMOVE, "GuiTheme");
+        entityEventDetails.put(EVENT_GUI_THEME_UPDATE, "GuiTheme");
+
+        // Extension
+        entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_UPDATE, ExtensionCustomAction.class);
+        entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_DELETE, ExtensionCustomAction.class);
     }
 
+    public static boolean isNetworkEvent(String eventType) {
+        return EVENT_NETWORK_CREATE.equals(eventType) || EVENT_NETWORK_DELETE.equals(eventType) ||
+                EVENT_NETWORK_UPDATE.equals(eventType);
+    }
     public static String getEntityForEvent(String eventName) {
         Object entityClass = entityEventDetails.get(eventName);
         if (entityClass == null) {
@@ -1219,4 +1419,12 @@ public class EventTypes {
 
         return null;
     }
+
+    public static boolean isVpcEvent(String eventType) {
+        return EventTypes.EVENT_VPC_CREATE.equals(eventType) || EventTypes.EVENT_VPC_DELETE.equals(eventType);
+    }
+
+    public static void addEntityEventDetail(String event, Class<?> clazz) {
+        entityEventDetails.put(event, clazz);
+    }
 }

api/src/main/java/com/cloud/exception/OperationTimedoutException.java

@@ -40,7 +40,7 @@ public class OperationTimedoutException extends CloudException {
     boolean _isActive;
 
     public OperationTimedoutException(Command[] cmds, long agentId, long seqId, int time, boolean isActive) {
-        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time);
+        super("Commands " + seqId + " to Host " + agentId + " timed out after " + time + " secs");
         _agentId = agentId;
         _seqId = seqId;
         _time = time;

api/src/main/java/com/cloud/exception/StorageAccessException.java

@@ -26,7 +26,7 @@ import com.cloud.utils.SerialVersionUID;
 public class StorageAccessException extends RuntimeException {
     private static final long serialVersionUID = SerialVersionUID.StorageAccessException;
 
-    public StorageAccessException(String message) {
-        super(message);
+    public StorageAccessException(String message, Exception causer) {
+        super(message, causer);
     }
 }

api/src/main/java/com/cloud/host/Host.java

@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.host;
 
+import com.cloud.cpu.CPU;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.resource.ResourceState;
 import com.cloud.utils.fsm.StateObject;
@@ -52,8 +53,12 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
             return strs;
         }
     }
-    public static final String HOST_UEFI_ENABLE = "host.uefi.enable";
-    public static final String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+
+    String HOST_UEFI_ENABLE = "host.uefi.enable";
+    String HOST_VOLUME_ENCRYPTION = "host.volume.encryption";
+    String HOST_INSTANCE_CONVERSION = "host.instance.conversion";
+    String HOST_OVFTOOL_VERSION = "host.ovftool.version";
+    String HOST_VIRTV2V_VERSION = "host.virtv2v.version";
 
     /**
      * @return name of the machine.
@@ -175,6 +180,8 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
      */
     Long getManagementServerId();
 
+    Long getLastManagementServerId();
+
     /*
      *@return removal date
      */
@@ -207,4 +214,8 @@ public interface Host extends StateObject<Status>, Identity, Partition, HAResour
     boolean isDisabled();
 
     ResourceState getResourceState();
+
+    CPU.CPUArch getArch();
+
+    String getStorageAccessGroups();
 }

api/src/main/java/com/cloud/host/Status.java

@@ -127,6 +127,7 @@ public enum Status {
         s_fsm.addTransition(Status.Connecting, Event.HostDown, Status.Down);
         s_fsm.addTransition(Status.Connecting, Event.Ping, Status.Connecting);
         s_fsm.addTransition(Status.Connecting, Event.ManagementServerDown, Status.Disconnected);
+        s_fsm.addTransition(Status.Connecting, Event.StartAgentRebalance, Status.Rebalancing);
         s_fsm.addTransition(Status.Connecting, Event.AgentDisconnected, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.PingTimeout, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.AgentDisconnected, Status.Alert);

api/src/main/java/com/cloud/hypervisor/Hypervisor.java

@@ -20,37 +20,58 @@ import com.cloud.storage.Storage.ImageFormat;
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
+import java.util.EnumSet;
+import java.util.stream.Collectors;
+
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.DirectDownloadTemplate;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.RootDiskSizeOverride;
+import static com.cloud.hypervisor.Hypervisor.HypervisorType.Functionality.VmStorageMigration;
 
 public class Hypervisor {
     public static class HypervisorType {
+        public enum Functionality {
+            DirectDownloadTemplate,
+            RootDiskSizeOverride,
+            VmStorageMigration
+        }
+
         private static final Map<String, HypervisorType> hypervisorTypeMap = new LinkedHashMap<>();
         public static final HypervisorType None = new HypervisorType("None"); //for storage hosts
-        public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD);
-        public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2);
-        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA);
+        public static final HypervisorType XenServer = new HypervisorType("XenServer", ImageFormat.VHD, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType KVM = new HypervisorType("KVM", ImageFormat.QCOW2, EnumSet.of(DirectDownloadTemplate, RootDiskSizeOverride, VmStorageMigration));
+        public static final HypervisorType VMware = new HypervisorType("VMware", ImageFormat.OVA, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
         public static final HypervisorType Hyperv = new HypervisorType("Hyperv");
         public static final HypervisorType VirtualBox = new HypervisorType("VirtualBox");
         public static final HypervisorType Parralels = new HypervisorType("Parralels");
         public static final HypervisorType BareMetal = new HypervisorType("BareMetal");
-        public static final HypervisorType Simulator = new HypervisorType("Simulator");
+        public static final HypervisorType Simulator = new HypervisorType("Simulator", null, EnumSet.of(RootDiskSizeOverride, VmStorageMigration));
         public static final HypervisorType Ovm = new HypervisorType("Ovm", ImageFormat.RAW);
         public static final HypervisorType Ovm3 = new HypervisorType("Ovm3", ImageFormat.RAW);
         public static final HypervisorType LXC = new HypervisorType("LXC");
-        public static final HypervisorType Custom = new HypervisorType("Custom");
+        public static final HypervisorType Custom = new HypervisorType("Custom", null, EnumSet.of(RootDiskSizeOverride));
+        public static final HypervisorType External = new HypervisorType("External", null, EnumSet.of(RootDiskSizeOverride));
         public static final HypervisorType Any = new HypervisorType("Any"); /*If you don't care about the hypervisor type*/
         private final String name;
         private final ImageFormat imageFormat;
+        private final Set<Functionality> supportedFunctionalities;
 
         public HypervisorType(String name) {
-            this(name, null);
+            this(name, null, EnumSet.noneOf(Functionality.class));
         }
 
         public HypervisorType(String name, ImageFormat imageFormat) {
+            this(name, imageFormat, EnumSet.noneOf(Functionality.class));
+        }
+
+        public HypervisorType(String name, ImageFormat imageFormat, Set<Functionality> supportedFunctionalities) {
             this.name = name;
             this.imageFormat = imageFormat;
+            this.supportedFunctionalities = supportedFunctionalities;
             if (name.equals("Parralels")){ // typo in the original code
                 hypervisorTypeMap.put("parallels", this);
             } else {
@@ -81,6 +102,12 @@ public class Hypervisor {
             return hypervisorType;
         }
 
+        public static List<HypervisorType> getListOfHypervisorsSupportingFunctionality(Functionality functionality) {
+            return hypervisorTypeMap.values().stream()
+                    .filter(hypervisor -> hypervisor.supportedFunctionalities.contains(functionality))
+                    .collect(Collectors.toList());
+        }
+
         /**
          * Returns the display name of a hypervisor type in case the custom hypervisor is used,
          * using the 'hypervisor.custom.display.name' setting. Otherwise, returns hypervisor name
@@ -102,6 +129,15 @@ public class Hypervisor {
             return name;
         }
 
+        /**
+         * Make this method to be part of the properties of the hypervisor type itself.
+         *
+         * @return true if the hypervisor plugin support the specified functionality
+         */
+        public boolean isFunctionalitySupported(Functionality functionality) {
+            return supportedFunctionalities.contains(functionality);
+        }
+
         @Override
         public int hashCode() {
             return Objects.hash(name);

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -23,6 +23,7 @@ import org.apache.cloudstack.backup.Backup;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
+import com.cloud.agent.api.to.DataStoreTO;
 import com.cloud.agent.api.to.NicTO;
 import com.cloud.agent.api.to.VirtualMachineTO;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
@@ -101,21 +102,20 @@ public interface HypervisorGuru extends Adapter {
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *
      * @param vm the stopped vm to migrate
-     * @param destination the primary storage pool to migrate to
+     * @param volumeToPool the primary storage pools to migrate to
      * @return a list of commands to perform for a successful migration
      */
     List<Command> finalizeMigrate(VirtualMachine vm, Map<Volume, StoragePool> volumeToPool);
 
 
     /**
-     * Will perform a clone of a VM on an external host (if the guru can handle)
+     * Will return the hypervisor VM (clone VM for PowerOn VMs), performs a clone of a VM if required on an external host (if the guru can handle)
      * @param hostIp VM's source host IP
-     * @param vmName name of the source VM to clone from
+     * @param vmName name of the source VM (clone VM name if cloned)
      * @param params hypervisor specific additional parameters
-     * @return a reference to the cloned VM
+     * @return a reference to the hypervisor or cloned VM, and cloned flag
      */
-    UnmanagedInstanceTO cloneHypervisorVMOutOfBand(String hostIp, String vmName,
-                                                   Map<String, String> params);
+    Pair<UnmanagedInstanceTO, Boolean> getHypervisorVMOutOfBandAndCloneIfRequired(String hostIp, String vmName, Map<String, String> params);
 
     /**
      * Removes a VM created as a clone of a VM on an external host
@@ -124,6 +124,23 @@ public interface HypervisorGuru extends Adapter {
      * @param params hypervisor specific additional parameters
      * @return true if the operation succeeds, false if not
      */
-    boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName,
-                                              Map<String, String> params);
+    boolean removeClonedHypervisorVMOutOfBand(String hostIp, String vmName, Map<String, String> params);
+
+    /**
+     * Create an OVA/OVF template of a VM on an external host (if the guru can handle)
+     * @param hostIp VM's source host IP
+     * @param vmName name of the source VM to create template from
+     * @param params hypervisor specific additional parameters
+     * @param templateLocation datastore to create the template file
+     * @return the created template dir/name
+     */
+    String createVMTemplateOutOfBand(String hostIp, String vmName, Map<String, String> params, DataStoreTO templateLocation, int threadsCountToExportOvf);
+
+    /**
+     * Removes the template on the location
+     * @param templateLocation datastore to remove the template file
+     * @param templateDir the template dir to remove from datastore
+     * @return true if the operation succeeds, false if not
+     */
+    boolean removeVMTemplateOutOfBand(DataStoreTO templateLocation, String templateDir);
 }

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesCluster.java

@@ -44,6 +44,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
         AutoscaleRequested,
         ScaleUpRequested,
         ScaleDownRequested,
+        AddNodeRequested,
+        RemoveNodeRequested,
         UpgradeRequested,
         OperationSucceeded,
         OperationFailed,
@@ -59,6 +61,8 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
         Stopped("All resources for the Kubernetes cluster are destroyed, Kubernetes cluster may still have ephemeral resource like persistent volumes provisioned"),
         Scaling("Transient state in which resources are either getting scaled up/down"),
         Upgrading("Transient state in which cluster is getting upgraded"),
+        Importing("Transient state in which additional nodes are added as worker nodes to a cluster"),
+        RemovingNodes("Transient state in which additional nodes are removed from a cluster"),
         Alert("State to represent Kubernetes clusters which are not in expected desired state (operationally in active control place, stopped cluster VM's etc)."),
         Recovering("State in which Kubernetes cluster is recovering from alert state"),
         Destroyed("End state of Kubernetes cluster in which all resources are destroyed, cluster will not be usable further"),
@@ -96,6 +100,17 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
             s_fsm.addTransition(State.Upgrading, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Upgrading, Event.OperationFailed, State.Alert);
 
+            s_fsm.addTransition(State.Running, Event.AddNodeRequested, State.Importing);
+            s_fsm.addTransition(State.Alert, Event.AddNodeRequested, State.Importing);
+            s_fsm.addTransition(State.Importing, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.Importing, Event.OperationFailed, State.Running);
+            s_fsm.addTransition(State.Alert, Event.OperationSucceeded, State.Running);
+
+            s_fsm.addTransition(State.Running, Event.RemoveNodeRequested, State.RemovingNodes);
+            s_fsm.addTransition(State.Alert, Event.RemoveNodeRequested, State.RemovingNodes);
+            s_fsm.addTransition(State.RemovingNodes, Event.OperationSucceeded, State.Running);
+            s_fsm.addTransition(State.RemovingNodes, Event.OperationFailed, State.Running);
+
             s_fsm.addTransition(State.Alert, Event.RecoveryRequested, State.Recovering);
             s_fsm.addTransition(State.Recovering, Event.OperationSucceeded, State.Running);
             s_fsm.addTransition(State.Recovering, Event.OperationFailed, State.Alert);
@@ -142,4 +157,13 @@ public interface KubernetesCluster extends ControlledEntity, com.cloud.utils.fsm
     Long getMaxSize();
     Long getSecurityGroupId();
     ClusterType getClusterType();
+    Long getControlNodeServiceOfferingId();
+    Long getWorkerNodeServiceOfferingId();
+    Long getEtcdNodeServiceOfferingId();
+    Long getControlNodeTemplateId();
+    Long getWorkerNodeTemplateId();
+    Long getEtcdNodeTemplateId();
+    Long getEtcdNodeCount();
+    Long getCniConfigId();
+    String getCniConfigDetails();
 }

api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.kubernetes.cluster;
+
+import org.apache.cloudstack.acl.ControlledEntity;
+
+import java.util.Map;
+
+import com.cloud.user.Account;
+import com.cloud.uservm.UserVm;
+import com.cloud.utils.component.Adapter;
+
+public interface KubernetesServiceHelper extends Adapter {
+
+    enum KubernetesClusterNodeType {
+        CONTROL, WORKER, ETCD, DEFAULT
+    }
+
+    ControlledEntity findByUuid(String uuid);
+    ControlledEntity findByVmId(long vmId);
+    void checkVmCanBeDestroyed(UserVm userVm);
+    boolean isValidNodeType(String nodeType);
+    Map<String, Long> getServiceOfferingNodeTypeMap(Map<String, Map<String, String>> serviceOfferingNodeTypeMap);
+    Map<String, Long> getTemplateNodeTypeMap(Map<String, Map<String, String>> templateNodeTypeMap);
+    void cleanupForAccount(Account account);
+}

api/src/main/java/com/cloud/network/IpAddress.java

@@ -99,4 +99,5 @@ public interface IpAddress extends ControlledEntity, Identity, InternalIdentity,
 
     boolean isForSystemVms();
 
+    boolean isForRouter();
 }

api/src/main/java/com/cloud/network/Ipv6Service.java

@@ -58,7 +58,7 @@ public interface Ipv6Service extends PluggableService, Configurable {
 
     Pair<Integer, Integer> getUsedTotalIpv6SubnetForZone(long zoneId);
 
-    Pair<String, String> preAllocateIpv6SubnetForNetwork(long zoneId) throws ResourceAllocationException;
+    Pair<String, String> preAllocateIpv6SubnetForNetwork(DataCenter zone) throws ResourceAllocationException;
 
     void assignIpv6SubnetToNetwork(String subnet, long networkId);
 

api/src/main/java/com/cloud/network/Network.java

@@ -103,7 +103,7 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
         public static final Service Vpn = new Service("Vpn", Capability.SupportedVpnProtocols, Capability.VpnTypes);
         public static final Service Dhcp = new Service("Dhcp", Capability.ExtraDhcpOptions);
         public static final Service Dns = new Service("Dns", Capability.AllowDnsSuffixModification);
-        public static final Service Gateway = new Service("Gateway");
+        public static final Service Gateway = new Service("Gateway", Capability.RedundantRouter);
         public static final Service Firewall = new Service("Firewall", Capability.SupportedProtocols, Capability.MultipleIps, Capability.TrafficStatistics,
                 Capability.SupportedTrafficDirection, Capability.SupportedEgressProtocols);
         public static final Service Lb = new Service("Lb", Capability.SupportedLBAlgorithms, Capability.SupportedLBIsolation, Capability.SupportedProtocols,
@@ -206,6 +206,7 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
         public static final Provider Tungsten = new Provider("Tungsten", false);
 
         public static final Provider Nsx = new Provider("Nsx", false);
+        public static final Provider Netris = new Provider("Netris", false);
 
         private final String name;
         private final boolean isExternal;
@@ -412,12 +413,16 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
 
     String getGateway();
 
+    void setGateway(String gateway);
+
     // "cidr" is the Cloudstack managed address space, all CloudStack managed vms get IP address from "cidr",
     // In general "cidr" also serves as the network CIDR
     // But in case IP reservation is configured for a Guest network, "networkcidr" is the Effective network CIDR for that network,
     // "cidr" will still continue to be the effective address space for CloudStack managed vms in that Guest network
     String getCidr();
 
+    void setCidr(String cidr);
+
     // "networkcidr" is the network CIDR of the guest network which uses IP reservation.
     // It is the summation of "cidr" and the reservedIPrange(the address space used for non CloudStack purposes).
     // For networks not configured with IP reservation, "networkcidr" is always null
@@ -503,4 +508,6 @@ public interface Network extends ControlledEntity, StateObject<Network.State>, I
     Integer getPublicMtu();
 
     Integer getPrivateMtu();
+
+    Integer getNetworkCidrSize();
 }

api/src/main/java/com/cloud/network/NetworkModel.java

@@ -149,7 +149,7 @@ public interface NetworkModel {
 
     boolean areServicesSupportedByNetworkOffering(long networkOfferingId, Service... services);
 
-    Network getNetworkWithSGWithFreeIPs(Long zoneId);
+    Network getNetworkWithSGWithFreeIPs(Account account, Long zoneId);
 
     Network getNetworkWithSecurityGroupEnabled(Long zoneId);
 
@@ -173,6 +173,8 @@ public interface NetworkModel {
 
     boolean isProviderSupportServiceInNetwork(long networkId, Service service, Provider provider);
 
+    boolean isAnyServiceSupportedInNetwork(long networkId, Provider provider, Service... services);
+
     boolean isProviderEnabledInPhysicalNetwork(long physicalNetowrkId, String providerName);
 
     String getNetworkTag(HypervisorType hType, Network network);
@@ -303,6 +305,8 @@ public interface NetworkModel {
 
     NicProfile getNicProfile(VirtualMachine vm, long networkId, String broadcastUri);
 
+    NicProfile getNicProfile(VirtualMachine vm, Nic nic, DataCenter dataCenter);
+
     Set<Long> getAvailableIps(Network network, String requestedIp);
 
     String getDomainNetworkDomain(long domainId, long zoneId);
@@ -317,6 +321,8 @@ public interface NetworkModel {
 
     void checkIp6Parameters(String startIPv6, String endIPv6, String ip6Gateway, String ip6Cidr) throws InvalidParameterValueException;
 
+    void checkIp6CidrSizeEqualTo64(String ip6Cidr) throws InvalidParameterValueException;
+
     void checkRequestedIpAddresses(long networkId, IpAddresses ips) throws InvalidParameterValueException;
 
     String getStartIpv6Address(long id);
@@ -354,4 +360,8 @@ public interface NetworkModel {
 
     void verifyIp6DnsPair(final String ip6Dns1, final String ip6Dns2);
 
+    boolean isSecurityGroupSupportedForZone(Long zoneId);
+
+    boolean checkSecurityGroupSupportForNetwork(Account account, DataCenter zone, List<Long> networkIds,
+                                                List<Long> securityGroupsIds);
 }

api/src/main/java/com/cloud/network/NetworkProfile.java

@@ -22,6 +22,7 @@ import java.util.Date;
 import com.cloud.network.Networks.BroadcastDomainType;
 import com.cloud.network.Networks.Mode;
 import com.cloud.network.Networks.TrafficType;
+import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
 
 public class NetworkProfile implements Network {
     private final long id;
@@ -41,8 +42,8 @@ public class NetworkProfile implements Network {
     private final Mode mode;
     private final BroadcastDomainType broadcastDomainType;
     private TrafficType trafficType;
-    private final String gateway;
-    private final String cidr;
+    private String gateway;
+    private String cidr;
     private final String networkCidr;
     private final String ip6Gateway;
     private final String ip6Cidr;
@@ -62,6 +63,7 @@ public class NetworkProfile implements Network {
     private final String guruName;
     private boolean strechedL2Subnet;
     private String externalId;
+    private Integer networkCidrSize;
 
     public NetworkProfile(Network network) {
         id = network.getId();
@@ -98,6 +100,7 @@ public class NetworkProfile implements Network {
         isRedundant = network.isRedundant();
         isRollingRestart = network.isRollingRestart();
         externalId = network.getExternalId();
+        networkCidrSize = network.getNetworkCidrSize();
     }
 
     @Override
@@ -210,11 +213,21 @@ public class NetworkProfile implements Network {
         return gateway;
     }
 
+    @Override
+    public void setGateway(String gateway) {
+        this.gateway = gateway;
+    }
+
     @Override
     public String getCidr() {
         return cidr;
     }
 
+    @Override
+    public void setCidr(String cidr) {
+        this.cidr = cidr;
+    }
+
     @Override
     public String getNetworkCidr() {
         return networkCidr;
@@ -367,4 +380,16 @@ public class NetworkProfile implements Network {
         return null;
     }
 
+    @Override
+    public Integer getNetworkCidrSize() {
+        return networkCidrSize;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("NetworkProfile %s",
+                ReflectionToStringBuilderUtils.reflectOnlySelectedFields(
+                        this, "id", "uuid", "name", "networkOfferingId"));
+    }
+
 }

api/src/main/java/com/cloud/network/NetworkService.java

@@ -19,7 +19,7 @@ package com.cloud.network;
 import java.util.List;
 import java.util.Map;
 
-import com.cloud.dc.DataCenter;
+import org.apache.cloudstack.acl.ControlledEntity;
 import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin;
 import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd;
 import org.apache.cloudstack.api.command.admin.network.ListDedicatedGuestVlanRangesCmd;
@@ -38,13 +38,16 @@ import org.apache.cloudstack.api.command.user.network.UpdateNetworkCmd;
 import org.apache.cloudstack.api.command.user.vm.ListNicsCmd;
 import org.apache.cloudstack.api.response.AcquirePodIpCmdResponse;
 import org.apache.cloudstack.framework.config.ConfigKey;
+import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 
+import com.cloud.agent.api.to.NicTO;
+import com.cloud.dc.DataCenter;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.exception.ResourceAllocationException;
 import com.cloud.exception.ResourceUnavailableException;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.network.Network.IpAddresses;
 import com.cloud.network.Network.Service;
 import com.cloud.network.Networks.TrafficType;
@@ -56,7 +59,6 @@ import com.cloud.utils.Pair;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.Nic;
 import com.cloud.vm.NicSecondaryIp;
-import org.apache.cloudstack.network.element.InternalLoadBalancerElementService;
 
 /**
  * The NetworkService interface is the "public" api to entities that make requests to the orchestration engine
@@ -102,6 +104,10 @@ public interface NetworkService {
 
     Network createGuestNetwork(CreateNetworkCmd cmd) throws InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException;
 
+    Network createGuestNetwork(long networkOfferingId, String name, String displayText, Account owner,
+           PhysicalNetwork physicalNetwork, long zoneId, ControlledEntity.ACLType aclType) throws
+            InsufficientCapacityException, ConcurrentOperationException, ResourceAllocationException;
+
     Pair<List<? extends Network>, Integer> searchForNetworks(ListNetworksCmd cmd);
 
     boolean deleteNetwork(long networkId, boolean forced);
@@ -263,4 +269,10 @@ public interface NetworkService {
     InternalLoadBalancerElementService getInternalLoadBalancerElementByNetworkServiceProviderId(long networkProviderId);
     InternalLoadBalancerElementService getInternalLoadBalancerElementById(long providerId);
     List<InternalLoadBalancerElementService> getInternalLoadBalancerElements();
+
+    boolean handleCksIsoOnNetworkVirtualRouter(Long virtualRouterId, boolean mount) throws ResourceUnavailableException;
+
+    IpAddresses getIpAddressesFromIps(String ipAddress, String ip6Address, String macAddress);
+
+    String getNicVlanValueForExternalVm(NicTO nic);
 }

api/src/main/java/com/cloud/network/Networks.java

@@ -129,7 +129,8 @@ public class Networks {
         UnDecided(null, null),
         OpenDaylight("opendaylight", String.class),
         TUNGSTEN("tf", String.class),
-        NSX("nsx", String.class);
+        NSX("nsx", String.class),
+        Netris("netris", String.class);
 
         private final String scheme;
         private final Class<?> type;

api/src/main/java/com/cloud/network/SDNProviderNetworkRule.java

@@ -0,0 +1,358 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network;
+
+import java.util.List;
+
+public class SDNProviderNetworkRule {
+
+    protected long domainId;
+    protected long accountId;
+    protected long zoneId;
+    protected Long networkResourceId;
+    protected String networkResourceName;
+    protected boolean isVpcResource;
+    protected long vmId;
+    protected long ruleId;
+    protected String publicIp;
+    protected String vmIp;
+    protected String publicPort;
+    protected String privatePort;
+    protected String protocol;
+    protected String algorithm;
+    protected List<String> sourceCidrList;
+    protected List<String> destinationCidrList;
+    protected Integer icmpCode;
+
+    protected Integer icmpType;
+    protected String trafficType;
+    protected Network.Service service;
+
+    public long getDomainId() {
+        return domainId;
+    }
+
+    public void setDomainId(long domainId) {
+        this.domainId = domainId;
+    }
+
+    public long getAccountId() {
+        return accountId;
+    }
+
+    public void setAccountId(long accountId) {
+        this.accountId = accountId;
+    }
+
+    public long getZoneId() {
+        return zoneId;
+    }
+
+    public void setZoneId(long zoneId) {
+        this.zoneId = zoneId;
+    }
+
+    public Long getNetworkResourceId() {
+        return networkResourceId;
+    }
+
+    public void setNetworkResourceId(Long networkResourceId) {
+        this.networkResourceId = networkResourceId;
+    }
+
+    public String getNetworkResourceName() {
+        return networkResourceName;
+    }
+
+    public void setNetworkResourceName(String networkResourceName) {
+        this.networkResourceName = networkResourceName;
+    }
+
+    public boolean isVpcResource() {
+        return isVpcResource;
+    }
+
+    public void setVpcResource(boolean vpcResource) {
+        isVpcResource = vpcResource;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public void setVmId(long vmId) {
+        this.vmId = vmId;
+    }
+
+    public long getRuleId() {
+        return ruleId;
+    }
+
+    public void setRuleId(long ruleId) {
+        this.ruleId = ruleId;
+    }
+
+    public String getPublicIp() {
+        return publicIp;
+    }
+
+    public void setPublicIp(String publicIp) {
+        this.publicIp = publicIp;
+    }
+
+    public String getVmIp() {
+        return vmIp;
+    }
+
+    public void setVmIp(String vmIp) {
+        this.vmIp = vmIp;
+    }
+
+    public String getPublicPort() {
+        return publicPort;
+    }
+
+    public void setPublicPort(String publicPort) {
+        this.publicPort = publicPort;
+    }
+
+    public String getPrivatePort() {
+        return privatePort;
+    }
+
+    public void setPrivatePort(String privatePort) {
+        this.privatePort = privatePort;
+    }
+
+    public String getProtocol() {
+        return protocol;
+    }
+
+    public void setProtocol(String protocol) {
+        this.protocol = protocol;
+    }
+
+    public void setAlgorithm(String algorithm) {
+        this.algorithm = algorithm;
+    }
+
+    public String getAlgorithm() {
+        return algorithm;
+    }
+
+    public Network.Service getService() {
+        return service;
+    }
+
+    public void setService(Network.Service service) {
+        this.service = service;
+    }
+
+    public Integer getIcmpCode() {
+        return icmpCode;
+    }
+
+    public void setIcmpCode(Integer icmpCode) {
+        this.icmpCode = icmpCode;
+    }
+
+    public Integer getIcmpType() {
+        return icmpType;
+    }
+
+    public void setIcmpType(Integer icmpType) {
+        this.icmpType = icmpType;
+    }
+
+    public List<String> getSourceCidrList() {
+        return sourceCidrList;
+    }
+
+    public void setSourceCidrList(List<String> sourceCidrList) {
+        this.sourceCidrList = sourceCidrList;
+    }
+
+    public List<String> getDestinationCidrList() {
+        return destinationCidrList;
+    }
+
+    public void setDestinationCidrList(List<String> destinationCidrList) {
+        this.destinationCidrList = destinationCidrList;
+    }
+
+    public String getTrafficType() {
+        return trafficType;
+    }
+
+    public void setTrafficType(String trafficType) {
+        this.trafficType = trafficType;
+    }
+
+    public static class Builder {
+        public long domainId;
+        public long accountId;
+        public long zoneId;
+        public Long networkResourceId;
+        public String networkResourceName;
+        public boolean isVpcResource;
+        public long vmId;
+
+        public long ruleId;
+        public String publicIp;
+        public String vmIp;
+        public String publicPort;
+        public String privatePort;
+        public String protocol;
+        public String algorithm;
+        public List<String> sourceCidrList;
+        public List<String> destinationCidrList;
+        public String trafficType;
+        public Integer icmpType;
+        public Integer icmpCode;
+        public Network.Service service;
+
+        public Builder() {
+            // Default constructor
+        }
+
+        public Builder setDomainId(long domainId) {
+            this.domainId = domainId;
+            return this;
+        }
+
+        public Builder setAccountId(long accountId) {
+            this.accountId = accountId;
+            return this;
+        }
+
+        public Builder setZoneId(long zoneId) {
+            this.zoneId = zoneId;
+            return this;
+        }
+
+        public Builder setNetworkResourceId(Long networkResourceId) {
+            this.networkResourceId = networkResourceId;
+            return this;
+        }
+
+        public Builder setNetworkResourceName(String networkResourceName) {
+            this.networkResourceName = networkResourceName;
+            return this;
+        }
+
+        public Builder setVpcResource(boolean isVpcResource) {
+            this.isVpcResource = isVpcResource;
+            return this;
+        }
+
+
+        public Builder setVmId(long vmId) {
+            this.vmId = vmId;
+            return this;
+        }
+
+        public Builder setRuleId(long ruleId) {
+            this.ruleId = ruleId;
+            return this;
+        }
+
+        public Builder setPublicIp(String publicIp) {
+            this.publicIp = publicIp;
+            return this;
+        }
+
+        public Builder setVmIp(String vmIp) {
+            this.vmIp = vmIp;
+            return this;
+        }
+
+        public Builder setPublicPort(String publicPort) {
+            this.publicPort = publicPort;
+            return this;
+        }
+
+        public Builder setPrivatePort(String privatePort) {
+            this.privatePort = privatePort;
+            return this;
+        }
+
+        public Builder setProtocol(String protocol) {
+            this.protocol = protocol;
+            return this;
+        }
+
+        public Builder setAlgorithm(String algorithm) {
+            this.algorithm = algorithm;
+            return this;
+        }
+
+        public Builder setTrafficType(String trafficType) {
+            this.trafficType = trafficType;
+            return this;
+        }
+
+        public Builder setIcmpType(Integer icmpType) {
+            this.icmpType = icmpType;
+            return this;
+        }
+
+        public Builder setIcmpCode(Integer icmpCode) {
+            this.icmpCode = icmpCode;
+            return this;
+        }
+
+        public Builder setSourceCidrList(List<String> sourceCidrList) {
+            this.sourceCidrList = sourceCidrList;
+            return this;
+        }
+
+        public Builder setDestinationCidrList(List<String> destinationCidrList) {
+            this.destinationCidrList = destinationCidrList;
+            return this;
+        }
+
+        public Builder setService(Network.Service service) {
+            this.service = service;
+            return this;
+        }
+
+        public SDNProviderNetworkRule build() {
+            SDNProviderNetworkRule rule = new SDNProviderNetworkRule();
+            rule.setDomainId(this.domainId);
+            rule.setAccountId(this.accountId);
+            rule.setZoneId(this.zoneId);
+            rule.setNetworkResourceId(this.networkResourceId);
+            rule.setNetworkResourceName(this.networkResourceName);
+            rule.setVpcResource(this.isVpcResource);
+            rule.setVmId(this.vmId);
+            rule.setVmIp(this.vmIp);
+            rule.setPublicIp(this.publicIp);
+            rule.setPublicPort(this.publicPort);
+            rule.setPrivatePort(this.privatePort);
+            rule.setProtocol(this.protocol);
+            rule.setRuleId(this.ruleId);
+            rule.setAlgorithm(this.algorithm);
+            rule.setIcmpType(this.icmpType);
+            rule.setIcmpCode(this.icmpCode);
+            rule.setSourceCidrList(this.sourceCidrList);
+            rule.setDestinationCidrList(this.destinationCidrList);
+            rule.setTrafficType(this.trafficType);
+            rule.setService(service);
+            return rule;
+        }
+    }
+}

api/src/main/java/com/cloud/network/Site2SiteVpnConnection.java

@@ -24,7 +24,7 @@ import org.apache.cloudstack.api.InternalIdentity;
 
 public interface Site2SiteVpnConnection extends ControlledEntity, InternalIdentity, Displayable {
     enum State {
-        Pending, Connecting, Connected, Disconnected, Error,
+        Pending, Connecting, Connected, Disconnected, Error, Removed
     }
 
     @Override

api/src/main/java/com/cloud/network/VirtualNetworkApplianceService.java

@@ -17,17 +17,22 @@
 package com.cloud.network;
 
 import java.util.List;
+import java.util.Map;
 
 import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd;
 import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd;
 
+import com.cloud.deploy.DeploymentPlanner;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.router.VirtualRouter;
 import com.cloud.user.Account;
 import com.cloud.utils.Pair;
 import com.cloud.vm.Nic;
+import com.cloud.vm.VirtualMachine;
+import com.cloud.vm.VirtualMachineProfile;
 
 public interface VirtualNetworkApplianceService {
     /**
@@ -62,6 +67,10 @@ public interface VirtualNetworkApplianceService {
 
     VirtualRouter startRouter(long id) throws ResourceUnavailableException, InsufficientCapacityException, ConcurrentOperationException;
 
+    void startRouterForHA(VirtualMachine vm, Map<VirtualMachineProfile.Param, Object> params, DeploymentPlanner planner)
+            throws InsufficientCapacityException, ResourceUnavailableException, ConcurrentOperationException,
+            OperationTimedoutException;
+
     VirtualRouter destroyRouter(long routerId, Account caller, Long callerUserId) throws ResourceUnavailableException, ConcurrentOperationException;
 
     VirtualRouter findRouter(long routerId);

api/src/main/java/com/cloud/network/VpcVirtualNetworkApplianceService.java

@@ -29,7 +29,6 @@ public interface VpcVirtualNetworkApplianceService extends VirtualNetworkApplian
     /**
      * @param router
      * @param network
-     * @param isRedundant
      * @param params TODO
      * @return
      * @throws ConcurrentOperationException
@@ -42,11 +41,30 @@ public interface VpcVirtualNetworkApplianceService extends VirtualNetworkApplian
     /**
      * @param router
      * @param network
-     * @param isRedundant
      * @return
      * @throws ConcurrentOperationException
      * @throws ResourceUnavailableException
      */
     boolean removeVpcRouterFromGuestNetwork(VirtualRouter router, Network network) throws ConcurrentOperationException, ResourceUnavailableException;
 
+
+    /**
+     * @param router
+     * @param network
+     * @return
+     * @throws ConcurrentOperationException
+     * @throws ResourceUnavailableException
+     */
+    boolean stopKeepAlivedOnRouter(VirtualRouter router, Network network) throws ConcurrentOperationException, ResourceUnavailableException;
+
+
+    /**
+     * @param router
+     * @param network
+     * @return
+     * @throws ConcurrentOperationException
+     * @throws ResourceUnavailableException
+     */
+    boolean startKeepAlivedOnRouter(VirtualRouter router, Network network) throws ConcurrentOperationException, ResourceUnavailableException;
+
 }

api/src/main/java/com/cloud/network/element/BgpServiceProvider.java

@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.element;
+
+import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.Network;
+import com.cloud.network.vpc.Vpc;
+
+import org.apache.cloudstack.network.BgpPeer;
+
+import java.util.List;
+
+public interface BgpServiceProvider extends NetworkElement {
+
+    boolean applyBgpPeers(Vpc vpc, Network network, List<? extends BgpPeer> bgpPeers) throws ResourceUnavailableException;
+
+}

api/src/main/java/com/cloud/network/element/LoadBalancingServiceProvider.java

@@ -48,4 +48,7 @@ public interface LoadBalancingServiceProvider extends NetworkElement, IpDeployin
     List<LoadBalancerTO> updateHealthChecks(Network network, List<LoadBalancingRule> lbrules);
 
     boolean handlesOnlyRulesInTransitionState();
+
+    default void expungeLbVmRefs(List<Long> vmIds, Long batchSize) {
+    }
 }

api/src/main/java/com/cloud/network/element/NetworkElement.java

@@ -23,6 +23,7 @@ import com.cloud.deploy.DeployDestination;
 import com.cloud.exception.ConcurrentOperationException;
 import com.cloud.exception.InsufficientCapacityException;
 import com.cloud.exception.ResourceUnavailableException;
+import com.cloud.network.IpAddress;
 import com.cloud.network.Network;
 import com.cloud.network.Network.Capability;
 import com.cloud.network.Network.Provider;
@@ -87,6 +88,14 @@ public interface NetworkElement extends Adapter {
     boolean release(Network network, NicProfile nic, VirtualMachineProfile vm, ReservationContext context) throws ConcurrentOperationException,
         ResourceUnavailableException;
 
+    /**
+     * Release IP from the network provider if reserved
+     * @param ipAddress
+     */
+    default boolean releaseIp(IpAddress ipAddress) {
+        return true;
+    }
+
     /**
      * The network is being shutdown.
      * @param network

api/src/main/java/com/cloud/network/element/PortForwardingServiceProvider.java

@@ -17,12 +17,40 @@
 package com.cloud.network.element;
 
 import java.util.List;
+import java.util.Objects;
 
 import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.network.Network;
+import com.cloud.network.rules.FirewallRule;
 import com.cloud.network.rules.PortForwardingRule;
+import com.cloud.network.vpc.NetworkACLItem;
 
 public interface PortForwardingServiceProvider extends NetworkElement, IpDeployingRequester {
+
+    static String getPublicPortRange(PortForwardingRule rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
+    static String getPrivatePFPortRange(PortForwardingRule rule) {
+        return rule.getDestinationPortStart() == rule.getDestinationPortEnd() ?
+                String.valueOf(rule.getDestinationPortStart()) :
+                String.valueOf(rule.getDestinationPortStart()).concat("-").concat(String.valueOf(rule.getDestinationPortEnd()));
+    }
+
+    static String getPrivatePortRange(FirewallRule rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
+    static String getPrivatePortRangeForACLRule(NetworkACLItem rule) {
+        return Objects.equals(rule.getSourcePortStart(), rule.getSourcePortEnd()) ?
+                String.valueOf(rule.getSourcePortStart()) :
+                String.valueOf(rule.getSourcePortStart()).concat("-").concat(String.valueOf(rule.getSourcePortEnd()));
+    }
+
     /**
      * Apply rules
      * @param network

api/src/main/java/com/cloud/network/element/VpcProvider.java

@@ -55,4 +55,8 @@ public interface VpcProvider extends NetworkElement {
     boolean applyACLItemsToPrivateGw(PrivateGateway gateway, List<? extends NetworkACLItem> rules) throws ResourceUnavailableException;
 
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+
+    default boolean updateVpc(Vpc vpc, String previousVpcName) {
+        return true;
+    }
 }

api/src/main/java/com/cloud/network/guru/NetworkGuru.java

@@ -212,4 +212,11 @@ public interface NetworkGuru extends Adapter {
 
     boolean isMyTrafficType(TrafficType type);
 
+    default boolean isSlaacV6Only() {
+        return true;
+    }
+
+    default boolean update(Network network, String prevNetworkName) {
+        return true;
+    }
 }

api/src/main/java/com/cloud/network/lb/LoadBalancingRule.java

@@ -63,6 +63,10 @@ public class LoadBalancingRule {
         return lb.getId();
     }
 
+    public LoadBalancer getLb() {
+        return lb;
+    }
+
     public String getName() {
         return lb.getName();
     }

api/src/main/java/com/cloud/network/netris/NetrisLbBackend.java

@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+public class NetrisLbBackend {
+    private long vmId;
+    private String vmIp;
+    private int port;
+
+    public NetrisLbBackend(long vmId, String vmIp, int port) {
+        this.vmId = vmId;
+        this.vmIp = vmIp;
+        this.port = port;
+    }
+
+    public long getVmId() {
+        return vmId;
+    }
+
+    public String getVmIp() {
+        return vmIp;
+    }
+
+    public int getPort() {
+        return port;
+    }
+}

api/src/main/java/com/cloud/network/netris/NetrisNetworkRule.java

@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import com.cloud.network.SDNProviderNetworkRule;
+
+
+import java.util.List;
+
+public class NetrisNetworkRule {
+    public enum NetrisRuleAction {
+        PERMIT, DENY
+    }
+
+    private SDNProviderNetworkRule baseRule;
+    private NetrisRuleAction aclAction;
+    private List<NetrisLbBackend> lbBackends;
+    private String lbRuleName;
+    private String lbCidrList;
+    private String reason;
+
+    public NetrisNetworkRule(Builder builder) {
+        this.baseRule = builder.baseRule;
+        this.aclAction = builder.aclAction;
+        this.lbBackends = builder.lbBackends;
+        this.reason = builder.reason;
+        this.lbCidrList = builder.lbCidrList;
+        this.lbRuleName = builder.lbRuleName;
+    }
+
+    public NetrisRuleAction getAclAction() {
+        return aclAction;
+    }
+
+    public List<NetrisLbBackend> getLbBackends() {
+        return lbBackends;
+    }
+
+    public String getReason() {
+        return reason;
+    }
+
+    public String getLbCidrList() {return lbCidrList; }
+
+    public String getLbRuleName() { return lbRuleName; }
+
+    public SDNProviderNetworkRule getBaseRule() {
+        return baseRule;
+    }
+
+    // Builder class extending the parent builder
+    public static class Builder {
+        private SDNProviderNetworkRule baseRule;
+        private NetrisRuleAction aclAction;
+        private List<NetrisLbBackend> lbBackends;
+        private String reason;
+        private String lbCidrList;
+        private String lbRuleName;
+
+        public Builder baseRule(SDNProviderNetworkRule baseRule) {
+            this.baseRule = baseRule;
+            return this;
+        }
+
+        public Builder aclAction(NetrisRuleAction aclAction) {
+            this.aclAction = aclAction;
+            return this;
+        }
+
+        public Builder lbBackends(List<NetrisLbBackend> lbBackends) {
+            this.lbBackends = lbBackends;
+            return this;
+        }
+
+        public Builder reason(String reason) {
+            this.reason = reason;
+            return this;
+        }
+
+        public Builder lbCidrList(String lbCidrList) {
+            this.lbCidrList = lbCidrList;
+            return this;
+        }
+
+        public Builder lbRuleName(String lbRuleName) {
+            this.lbRuleName = lbRuleName;
+            return this;
+        }
+
+        public NetrisNetworkRule build() {
+            return new NetrisNetworkRule(this);
+        }
+    }
+}

api/src/main/java/com/cloud/network/netris/NetrisProvider.java

@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import org.apache.cloudstack.api.Identity;
+import org.apache.cloudstack.api.InternalIdentity;
+
+public interface NetrisProvider extends InternalIdentity, Identity {
+    long getZoneId();
+    String getName();
+    String getUrl();
+    String getUsername();
+    String getSiteName();
+    String getTenantName();
+    String getNetrisTag();
+}

api/src/main/java/com/cloud/network/netris/NetrisService.java

@@ -0,0 +1,310 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.network.netris;
+
+import com.cloud.network.IpAddress;
+import com.cloud.network.Network;
+import com.cloud.network.SDNProviderNetworkRule;
+import com.cloud.network.vpc.StaticRoute;
+import com.cloud.network.vpc.Vpc;
+
+import java.util.List;
+
+/**
+ * Interface for Netris Services that provides methods to manage VPCs, networks,
+ * NAT rules, network rules, and static routes in an SDN (Software Defined Networking) environment.
+ */
+
+public interface NetrisService {
+
+    /**
+     * Creates IPAM (IP Address Management) allocations for zone-level public ranges.
+     *
+     * @param zoneId the ID of the zone
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createIPAMAllocationsForZoneLevelPublicRanges(long zoneId);
+
+    /**
+     * Creates a VPC (Virtual Private Cloud) resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcId            the ID of the VPC
+     * @param vpcName          the name of the VPC
+     * @param sourceNatEnabled true if source NAT is enabled
+     * @param cidr             the CIDR of the VPC
+     * @param isVpcNetwork     true if it is a VPC network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createVpcResource(long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled, String cidr, boolean isVpcNetwork);
+
+    /**
+     * Updates an existing VPC resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcId            the ID of the VPC
+     * @param vpcName          the new name of the VPC
+     * @param previousVpcName  the previous name of the VPC
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVpcResource(long zoneId, long accountId, long domainId, Long vpcId, String vpcName, String previousVpcName);
+
+    /**
+     * Deletes a VPC resource.
+     *
+     * @param zoneId   the ID of the zone
+     * @param accountId the ID of the account
+     * @param domainId the ID of the domain
+     * @param vpc      the VPC to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteVpcResource(long zoneId, long accountId, long domainId, Vpc vpc);
+
+    /**
+     * Creates a virtual network (vNet) resource.
+     *
+     * @param zoneId           the ID of the zone
+     * @param accountId        the ID of the account
+     * @param domainId         the ID of the domain
+     * @param vpcName          the name of the VPC
+     * @param vpcId            the ID of the VPC
+     * @param networkName      the name of the network
+     * @param networkId        the ID of the network
+     * @param cidr             the CIDR of the network
+     * @param globalRouting    true if global routing is enabled
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr, Boolean globalRouting);
+
+    /**
+     * Updates an existing vNet resource.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the new name of the network
+     * @param networkId       the ID of the network
+     * @param prevNetworkName the previous name of the network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String prevNetworkName);
+
+    /**
+     * Deletes an existing vNet resource.
+     *
+     * @param zoneId       the ID of the zone
+     * @param accountId    the ID of the account
+     * @param domainId     the ID of the domain
+     * @param vpcName      the name of the VPC
+     * @param vpcId        the ID of the VPC
+     * @param networkName  the name of the network
+     * @param networkId    the ID of the network
+     * @param cidr         the CIDR of the network
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteVnetResource(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr);
+
+    /**
+     * Creates a source NAT rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param sourceNatIp     the source NAT IP
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createSnatRule(long zoneId, long accountId, long domainId, String vpcName, long vpcId, String networkName, long networkId, boolean isForVpc, String vpcCidr, String sourceNatIp);
+
+    /**
+     * Creates a port forwarding rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param networkRule     the network rule to forward
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createPortForwardingRule(long zoneId, long accountId, long domainId, String vpcName, long vpcId, String networkName, Long networkId, boolean isForVpc, String vpcCidr, SDNProviderNetworkRule networkRule);
+
+    /**
+     * Deletes a port forwarding rule for a VPC or network.
+     *
+     * @param zoneId          the ID of the zone
+     * @param accountId       the ID of the account
+     * @param domainId        the ID of the domain
+     * @param vpcName         the name of the VPC
+     * @param vpcId           the ID of the VPC
+     * @param networkName     the name of the network
+     * @param networkId       the ID of the network
+     * @param isForVpc        true if the rule applies to a VPC
+     * @param vpcCidr         the VPC CIDR
+     * @param networkRule     the network rule to remove
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deletePortForwardingRule(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, boolean isForVpc, String vpcCidr, SDNProviderNetworkRule networkRule);
+
+    /**
+     * Updates the source NAT IP for a specified VPC.
+     *
+     * @param vpc     the VPC to updates
+     * @param address the new source NAT IP address
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+
+    /**
+     * Creates a static NAT rule for a specific VM.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if the rule applies to a VPC
+     * @param vpcCidr            the VPC CIDR
+     * @param staticNatIp        the static NAT IP
+     * @param vmIp               the VM's IP address
+     * @param vmId               the ID of the VM
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createStaticNatRule(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String vpcCidr, String staticNatIp, String vmIp, long vmId);
+
+    /**
+     * Deletes a static NAT rule for a specific VM.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if the rule applies to a VPC
+     * @param staticNatIp        the static NAT IP
+     * @param vmId               the ID of the VM
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteStaticNatRule(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String staticNatIp, long vmId);
+
+    /**
+     * Adds firewall rules to a specific network.
+     *
+     * @param network       the target network
+     * @param firewallRules the list of firewall rules to add
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean addFirewallRules(Network network, List<NetrisNetworkRule> firewallRules);
+
+    /**
+     * Deletes firewall rules from a specific network.
+     *
+     * @param network       the target network
+     * @param firewallRules the list of firewall rules to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteFirewallRules(Network network, List<NetrisNetworkRule> firewallRules);
+
+    /**
+     * Adds or updates a static route for a specific network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @param updateRoute        true if the route should be updated
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean addOrUpdateStaticRoute(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId, boolean updateRoute);
+
+    /**
+     * Deletes a specific static route for a network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteStaticRoute(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId);
+
+    /**
+     * Lists static routes for a specific network or VPC.
+     *
+     * @param zoneId             the ID of the zone
+     * @param accountId          the ID of the account
+     * @param domainId           the ID of the domain
+     * @param networkResourceName the name of the network resource
+     * @param networkResourceId  the ID of the network resource
+     * @param isForVpc           true if it is for a VPC
+     * @param prefix             the IP prefix of the route
+     * @param nextHop            the next hop address
+     * @param routeId            the ID of the route
+     * @return a list of static routes
+     */
+    List<StaticRoute> listStaticRoutes(long zoneId, long accountId, long domainId, String networkResourceName, Long networkResourceId, boolean isForVpc, String prefix, String nextHop, Long routeId);
+
+    /**
+     * Releases a NAT IP address.
+     *
+     * @param zoneId   the ID of the zone
+     * @param publicIp the public NAT IP to release
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean releaseNatIp(long zoneId, String publicIp);
+
+    /**
+     * Creates or updates a load balancer (LB) rule.
+     *
+     * @param rule the network rule for the load balancer
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean createOrUpdateLbRule(NetrisNetworkRule rule);
+
+    /**
+     * Deletes a load balancer (LB) rule.
+     *
+     * @param rule the network rule to delete
+     * @return true if the operation is successful, false otherwise
+     */
+    boolean deleteLbRule(NetrisNetworkRule rule);
+}

api/src/main/java/com/cloud/network/nsx/NsxService.java

@@ -16,11 +16,23 @@
 // under the License.
 package com.cloud.network.nsx;
 
+import org.apache.cloudstack.framework.config.ConfigKey;
+
 import com.cloud.network.IpAddress;
 import com.cloud.network.vpc.Vpc;
 
 public interface NsxService {
 
+    ConfigKey<Integer> NSX_API_FAILURE_RETRIES = new ConfigKey<>("Advanced", Integer.class,
+            "nsx.api.failure.retries", "30",
+            "Number of retries for NSX API operations in case of failures",
+            true, ConfigKey.Scope.Zone);
+    ConfigKey<Integer> NSX_API_FAILURE_INTERVAL = new ConfigKey<>("Advanced", Integer.class,
+            "nsx.api.failure.interval", "60",
+            "Waiting time (in seconds) before retrying an NSX API operation in case of failure",
+            true, ConfigKey.Scope.Zone);
+
     boolean createVpcNetwork(Long zoneId, long accountId, long domainId, Long vpcId, String vpcName, boolean sourceNatEnabled);
     boolean updateVpcSourceNatIp(Vpc vpc, IpAddress address);
+    String getSegmentId(long domainId, long accountId, long zoneId, Long vpcId, long networkId);
 }

api/src/main/java/com/cloud/network/rules/RulesService.java

@@ -26,6 +26,7 @@ import com.cloud.exception.ResourceUnavailableException;
 import com.cloud.user.Account;
 import com.cloud.utils.Pair;
 import com.cloud.utils.net.Ip;
+import org.apache.cloudstack.api.command.user.firewall.UpdatePortForwardingRuleCmd;
 
 public interface RulesService {
     Pair<List<? extends FirewallRule>, Integer> searchStaticNatRules(Long ipId, Long id, Long vmId, Long start, Long size, String accountName, Long domainId,
@@ -81,6 +82,8 @@ public interface RulesService {
 
     boolean disableStaticNat(long ipId) throws ResourceUnavailableException, NetworkRuleConflictException, InsufficientAddressCapacityException;
 
-    PortForwardingRule updatePortForwardingRule(long id, Integer privatePort, Integer privateEndPort, Long virtualMachineId, Ip vmGuestIp, String customId, Boolean forDisplay);
+    PortForwardingRule updatePortForwardingRule(UpdatePortForwardingRuleCmd cmd);
+
+    void  validatePortForwardingSourceCidrList(List<String> sourceCidrList);
 
 }

api/src/main/java/com/cloud/network/vpc/StaticRoute.java

@@ -25,6 +25,7 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit
         Staged, // route been created but has never got through network rule conflict detection.  Routes in this state can not be sent to VPC virtual router.
         Add,    // Add means the route has been created and has gone through network rule conflict detection.
         Active, // Route has been sent to the VPC router and reported to be active.
+        Update,
         Revoke,  // Revoke means this route has been revoked. If this route has been sent to the VPC router, the route will be deleted from database.
         Deleting // rule has been revoked and is scheduled for deletion
     }
@@ -32,7 +33,9 @@ public interface StaticRoute extends ControlledEntity, Identity, InternalIdentit
     /**
      * @return
      */
-    long getVpcGatewayId();
+    Long getVpcGatewayId();
+
+    String getNextHop();
 
     /**
      * @return

api/src/main/java/com/cloud/network/vpc/StaticRouteProfile.java

@@ -23,7 +23,8 @@ public class StaticRouteProfile implements StaticRoute {
     private String targetCidr;
     private long accountId;
     private long domainId;
-    private long gatewayId;
+    private Long gatewayId;
+    private String nextHop;
     private StaticRoute.State state;
     private long vpcId;
     String vlanTag;
@@ -46,6 +47,18 @@ public class StaticRouteProfile implements StaticRoute {
         ipAddress = gateway.getIp4Address();
     }
 
+    public StaticRouteProfile(StaticRoute staticRoute) {
+        id = staticRoute.getId();
+        uuid = staticRoute.getUuid();
+        targetCidr = staticRoute.getCidr();
+        accountId = staticRoute.getAccountId();
+        domainId = staticRoute.getDomainId();
+        gatewayId = staticRoute.getVpcGatewayId();
+        state = staticRoute.getState();
+        vpcId = staticRoute.getVpcId();
+        gateway = staticRoute.getNextHop();
+    }
+
     @Override
     public long getAccountId() {
         return accountId;
@@ -57,10 +70,15 @@ public class StaticRouteProfile implements StaticRoute {
     }
 
     @Override
-    public long getVpcGatewayId() {
+    public Long getVpcGatewayId() {
         return gatewayId;
     }
 
+    @Override
+    public String getNextHop() {
+        return nextHop;
+    }
+
     @Override
     public String getCidr() {
         return targetCidr;