+
To support incremental migration from one version to another without having to redeploy the database, the CloudStack supports an incremental schema migration mechanism for the database.
+!!!How does it work?
+When the database is deployed for the first time with [[waf deploydb]] or the command {{{cloud-setup-databases}}}, a row is written to the {{{configuration}}} table, named {{{schema.level}}} and containing the current schema level. This schema level row comes from the file {{{setup/db/schema-level.sql}}} in the source (refer to the [[Installation paths]] topic to find out where this file is installed in a running system).
+
+This value is used by the database migrator {{{cloud-migrate-databases}}} (source {{{setup/bindir/cloud-migrate-databases.in}}}) to determine the starting schema level. The database migrator has a series of classes -- each class represents a step in the migration process and is usually tied to the execution of a SQL file stored in {{{setup/db}}}. To migrate the database, the database migrator:
+# walks the list of steps it knows about,
+# generates a list of steps sorted by the order they should be executed in,
+# executes each step in order
+# at the end of each step, records the new schema level to the database table {{{configuration}}}
+For more information, refer to the database migrator source -- it is documented.
+!!!What impact does this have on me as a developer?
+Whenever you need to evolve the schema of the database:
+# write a migration SQL script and store it in {{{setup/db}}},
+# include your schema changes in the appropriate SQL file {{{create-*.sql}}} too (as the database is expected to be at its latest evolved schema level right after deploying a fresh database)
+# write a class in {{{setup/bindir/cloud-migrate-databases.in}}}, describing the migration step; in detail:
+## the schema level your migration step expects the database to be in,
+## the schema level your migration step will leave your database in (presumably the latest schema level, which you will have to choose!),
+## and the name / description of the step
+# bump the schema level in {{{setup/db/schema-level.sql}}} to the latest schema level
+Otherwise, ''end-user migration will fail catastrophically''.
+
+
Start here if you want to learn the essentials to extend, modify and enhance the CloudStack. This assumes that you've already familiarized yourself with CloudStack concepts, installation and configuration using the [[Getting started|Welcome]] instructions.
* [[Obtain the source|Obtaining the source]]
* [[Prepare your environment|Preparing your development environment]]
@@ -764,6 +786,7 @@ Extra developer information:
* [[How to integrate with Eclipse]]
* [[Starting over]]
* [[Making a source release|waf dist]]
+* [[How to write database migration scripts|Database migration infrastructure]]
diff --git a/cloud.spec b/cloud.spec
index 690c816876e..8207e103408 100644
--- a/cloud.spec
+++ b/cloud.spec
@@ -582,6 +582,9 @@ fi
%{_datadir}/%{name}/setup/index-212to213.sql
%{_datadir}/%{name}/setup/postprocess-20to21.sql
%{_datadir}/%{name}/setup/schema-20to21.sql
+%{_datadir}/%{name}/setup/schema-level.sql
+%{_datadir}/%{name}/setup/schema-21to22.sql
+%{_datadir}/%{name}/setup/data-21to22.sql
%doc README
%doc INSTALL
%doc HACKING
diff --git a/debian/cloud-setup.install b/debian/cloud-setup.install
index 7d35dbe9929..48969370521 100644
--- a/debian/cloud-setup.install
+++ b/debian/cloud-setup.install
@@ -12,3 +12,6 @@
/usr/share/cloud/setup/index-212to213.sql
/usr/share/cloud/setup/postprocess-20to21.sql
/usr/share/cloud/setup/schema-20to21.sql
+/usr/share/cloud/setup/schema-level.sql
+/usr/share/cloud/setup/schema-21to22.sql
+/usr/share/cloud/setup/data-21to22.sql
diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java
index a1ce58b1599..053bd0484a4 100644
--- a/server/src/com/cloud/server/ManagementServerImpl.java
+++ b/server/src/com/cloud/server/ManagementServerImpl.java
@@ -101,15 +101,15 @@ import com.cloud.async.executor.SecurityGroupParam;
import com.cloud.async.executor.UpdateLoadBalancerParam;
import com.cloud.async.executor.UpgradeVMParam;
import com.cloud.async.executor.VMOperationParam;
-import com.cloud.async.executor.VMOperationParam.VmOp;
import com.cloud.async.executor.VolumeOperationParam;
+import com.cloud.async.executor.VMOperationParam.VmOp;
import com.cloud.async.executor.VolumeOperationParam.VolumeOp;
import com.cloud.capacity.CapacityVO;
import com.cloud.capacity.dao.CapacityDao;
import com.cloud.configuration.ConfigurationManager;
import com.cloud.configuration.ConfigurationVO;
-import com.cloud.configuration.ResourceCount.ResourceType;
import com.cloud.configuration.ResourceLimitVO;
+import com.cloud.configuration.ResourceCount.ResourceType;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.configuration.dao.ResourceLimitDao;
import com.cloud.consoleproxy.ConsoleProxyManager;
@@ -119,8 +119,8 @@ import com.cloud.dc.DataCenterIpAddressVO;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.HostPodVO;
import com.cloud.dc.PodVlanMapVO;
-import com.cloud.dc.Vlan.VlanType;
import com.cloud.dc.VlanVO;
+import com.cloud.dc.Vlan.VlanType;
import com.cloud.dc.dao.AccountVlanMapDao;
import com.cloud.dc.dao.ClusterDao;
import com.cloud.dc.dao.DataCenterDao;
@@ -175,7 +175,6 @@ import com.cloud.network.security.NetworkGroupRulesVO;
import com.cloud.network.security.NetworkGroupVO;
import com.cloud.network.security.dao.NetworkGroupDao;
import com.cloud.offering.NetworkOffering;
-import com.cloud.offering.NetworkOffering.GuestIpType;
import com.cloud.offering.ServiceOffering;
import com.cloud.serializer.GsonHelper;
import com.cloud.server.auth.UserAuthenticator;
@@ -188,13 +187,10 @@ import com.cloud.storage.GuestOSCategoryVO;
import com.cloud.storage.GuestOSVO;
import com.cloud.storage.LaunchPermissionVO;
import com.cloud.storage.Snapshot;
-import com.cloud.storage.Snapshot.SnapshotType;
import com.cloud.storage.SnapshotPolicyVO;
import com.cloud.storage.SnapshotScheduleVO;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.FileSystem;
-import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePoolHostVO;
import com.cloud.storage.StoragePoolVO;
@@ -202,9 +198,12 @@ import com.cloud.storage.StorageStats;
import com.cloud.storage.VMTemplateHostVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
-import com.cloud.storage.Volume.VolumeType;
import com.cloud.storage.VolumeStats;
import com.cloud.storage.VolumeVO;
+import com.cloud.storage.Snapshot.SnapshotType;
+import com.cloud.storage.Storage.FileSystem;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.Volume.VolumeType;
import com.cloud.storage.dao.DiskOfferingDao;
import com.cloud.storage.dao.DiskTemplateDao;
import com.cloud.storage.dao.GuestOSCategoryDao;
@@ -215,9 +214,9 @@ import com.cloud.storage.dao.SnapshotPolicyDao;
import com.cloud.storage.dao.StoragePoolDao;
import com.cloud.storage.dao.StoragePoolHostDao;
import com.cloud.storage.dao.VMTemplateDao;
-import com.cloud.storage.dao.VMTemplateDao.TemplateFilter;
import com.cloud.storage.dao.VMTemplateHostDao;
import com.cloud.storage.dao.VolumeDao;
+import com.cloud.storage.dao.VMTemplateDao.TemplateFilter;
import com.cloud.storage.preallocatedlun.PreallocatedLunVO;
import com.cloud.storage.preallocatedlun.dao.PreallocatedLunDao;
import com.cloud.storage.secondary.SecondaryStorageVmManager;
@@ -239,12 +238,12 @@ import com.cloud.user.dao.UserDao;
import com.cloud.user.dao.UserStatisticsDao;
import com.cloud.uservm.UserVm;
import com.cloud.utils.DateUtil;
-import com.cloud.utils.DateUtil.IntervalType;
import com.cloud.utils.EnumUtils;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.Pair;
import com.cloud.utils.PasswordGenerator;
import com.cloud.utils.StringUtils;
+import com.cloud.utils.DateUtil.IntervalType;
import com.cloud.utils.component.Adapters;
import com.cloud.utils.component.ComponentLocator;
import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -921,6 +920,9 @@ public class ManagementServerImpl implements ManagementServer {
// Mark the account's volumes as destroyed
List volumes = _volumeDao.findDetachedByAccount(accountId);
for (VolumeVO volume : volumes) {
+ if(volume.getPoolId()==null){
+ accountCleanupNeeded = true;
+ }
_storageMgr.destroyVolume(volume);
}
diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
index c6f82d2f567..d2cd55e2c68 100644
--- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
+++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java
@@ -1006,6 +1006,9 @@ public class SnapshotManagerImpl implements SnapshotManager {
// i.e Call them before the VMs for those volumes are destroyed.
boolean success = true;
for (VolumeVO volume : volumes) {
+ if(volume.getPoolId()==null){
+ continue;
+ }
Long volumeId = volume.getId();
Long dcId = volume.getDataCenterId();
String secondaryStoragePoolURL = _storageMgr.getSecondaryStorageURL(dcId);
diff --git a/setup/bindir/cloud-migrate-databases.in b/setup/bindir/cloud-migrate-databases.in
index dab1515d50c..6adffa75d3d 100644
--- a/setup/bindir/cloud-migrate-databases.in
+++ b/setup/bindir/cloud-migrate-databases.in
@@ -150,6 +150,23 @@ class From21datamigratedTo21postprocessed(cloud_utils.MigrationStep):
to_level = "2.1"
def run(self): self.context.run_sql_resource("postprocess-20to21.sql")
+class From21To213(cloud_utils.MigrationStep):
+ def __str__(self): return "Dropping obsolete indexes"
+ from_level = "2.1"
+ to_level = "2.1.3"
+ def run(self): self.context.run_sql_resource("index-212to213.sql")
+
+class From213To22data(cloud_utils.MigrationStep):
+ def __str__(self): return "Migrating data"
+ from_level = "2.1.3"
+ to_level = "2.2-01"
+ def run(self): self.context.run_sql_resource("data-21to22.sql")
+
+class From22dataTo22(cloud_utils.MigrationStep):
+ def __str__(self): return "Migrating indexes"
+ from_level = "2.2-01"
+ to_level = "2.2"
+ def run(self): self.context.run_sql_resource("index-21to22.sql")
# command line harness functions
diff --git a/setup/bindir/cloud-setup-databases.in b/setup/bindir/cloud-setup-databases.in
index 6c34575fcc0..3fe66d9c111 100755
--- a/setup/bindir/cloud-setup-databases.in
+++ b/setup/bindir/cloud-setup-databases.in
@@ -352,3 +352,10 @@ if rootuser:
print "Applying file %s to the database on server %s:%s"%(p,host,port)
try: run_mysql(text,rootuser,rootpassword,host,port)
except CalledProcessError: sys.exit(22)
+
+ p = os.path.join(dbfilepath,"schema-level.sql")
+ if os.path.isfile(p):
+ text = file(p).read()
+ print "Applying file %s to the database on server %s:%s"%(p,host,port)
+ try: run_mysql(text,rootuser,rootpassword,host,port)
+ except CalledProcessError: sys.exit(22)
diff --git a/setup/db/migration/data-21to22.sql b/setup/db/data-21to22.sql
similarity index 100%
rename from setup/db/migration/data-21to22.sql
rename to setup/db/data-21to22.sql
diff --git a/setup/db/migration/schema-21to22.sql b/setup/db/schema-21to22.sql
similarity index 100%
rename from setup/db/migration/schema-21to22.sql
rename to setup/db/schema-21to22.sql
diff --git a/setup/db/schema-level.sql b/setup/db/schema-level.sql
new file mode 100644
index 00000000000..e3b0eea48fb
--- /dev/null
+++ b/setup/db/schema-level.sql
@@ -0,0 +1 @@
+INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) VALUES ('Hidden', 'DEFAULT', 'database', 'schema.level', '2.2', 'The schema level of this database');
diff --git a/ui/test/scripts/cloud.core.test.js b/ui/test/scripts/cloud.core.test.js
index e9c61562647..0aacfd85a66 100644
--- a/ui/test/scripts/cloud.core.test.js
+++ b/ui/test/scripts/cloud.core.test.js
@@ -290,7 +290,7 @@ $(document).ready(function() {
submenuContent.find("#grid_content").prepend(template.fadeIn("slow"));
var username = thisDialog.find("#add_user_username").val();
- var password = $.md5(thisDialog.find("#add_user_password").val());
+ var password = $.md5(encodeURIComponent(thisDialog.find("#add_user_password").val()));
var email = thisDialog.find("#add_user_email").val();
if(email == "")
email = username;
@@ -318,7 +318,7 @@ $(document).ready(function() {
$.ajax({
type: "POST",
- data: createURL("command=createUser&username="+encodeURIComponent(username)+"&password="+encodeURIComponent(password)+"&email="+encodeURIComponent(email)+"&firstname="+encodeURIComponent(firstname)+"&lastname="+encodeURIComponent(lastname)+"&account="+account+"&accounttype="+accountType+"&domainid="+domainId+moreCriteria.join("")+"&response=json"),
+ data: createURL("command=createUser&username="+encodeURIComponent(username)+"&password="+password+"&email="+encodeURIComponent(email)+"&firstname="+encodeURIComponent(firstname)+"&lastname="+encodeURIComponent(lastname)+"&account="+account+"&accounttype="+accountType+"&domainid="+domainId+moreCriteria.join("")+"&response=json"),
dataType: "json",
async: false,
success: function(json) {
diff --git a/wscript b/wscript
index 9286ded759a..e04ee7e8980 100644
--- a/wscript
+++ b/wscript
@@ -777,6 +777,12 @@ def deploydb(ctx,virttech=None):
after = after + file(p).read()
Utils.pprint("GREEN","Reading database code from %s"%p)
+ p = _join("setup","db","schema-level.sql")
+ if _exists(p):
+ p = dev_override(p)
+ after = after + file(p).read()
+ Utils.pprint("GREEN","Reading database code from %s"%p)
+
cmd = ["mysql","--user=%s"%dbuser,"-h",dbhost,"--password=%s"%dbpw]
Utils.pprint("GREEN","Deploying post-configuration database scripts to %s (user %s)"%(dbhost,dbuser))
Utils.pprint("BLUE"," ".join(cmd))