From 1d7e376b1f850b54bf29125b55084e57b8d0f279 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 10:59:31 -0700 Subject: [PATCH 01/11] Move migration/*sql to setup/db/ where migration tool needs them --- setup/db/{migration => }/data-21to22.sql | 0 setup/db/{migration => }/schema-21to22.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename setup/db/{migration => }/data-21to22.sql (100%) rename setup/db/{migration => }/schema-21to22.sql (100%) diff --git a/setup/db/migration/data-21to22.sql b/setup/db/data-21to22.sql similarity index 100% rename from setup/db/migration/data-21to22.sql rename to setup/db/data-21to22.sql diff --git a/setup/db/migration/schema-21to22.sql b/setup/db/schema-21to22.sql similarity index 100% rename from setup/db/migration/schema-21to22.sql rename to setup/db/schema-21to22.sql From ae942449cf3449df8b21497b38f0216b5eb90e60 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 10:59:56 -0700 Subject: [PATCH 02/11] Add 2.1.3 migration step --- setup/bindir/cloud-migrate-databases.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup/bindir/cloud-migrate-databases.in b/setup/bindir/cloud-migrate-databases.in index dab1515d50c..5c6abd86698 100644 --- a/setup/bindir/cloud-migrate-databases.in +++ b/setup/bindir/cloud-migrate-databases.in @@ -150,6 +150,11 @@ class From21datamigratedTo21postprocessed(cloud_utils.MigrationStep): to_level = "2.1" def run(self): self.context.run_sql_resource("postprocess-20to21.sql") +class From21To213(cloud_utils.MigrationStep): + def __str__(self): return "Dropping obsolete indexes" + from_level = "2.1" + to_level = "2.1.3" + def run(self): self.context.run_sql_resource("index-212to213.sql") # command line harness functions From 98c00de7ff07b06fba73cffd076a09dbdf6b4bb2 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 11:00:05 -0700 Subject: [PATCH 03/11] Add 2.2 migration steps --- setup/bindir/cloud-migrate-databases.in | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup/bindir/cloud-migrate-databases.in b/setup/bindir/cloud-migrate-databases.in index 5c6abd86698..6adffa75d3d 100644 --- a/setup/bindir/cloud-migrate-databases.in +++ b/setup/bindir/cloud-migrate-databases.in @@ -156,6 +156,18 @@ class From21To213(cloud_utils.MigrationStep): to_level = "2.1.3" def run(self): self.context.run_sql_resource("index-212to213.sql") +class From213To22data(cloud_utils.MigrationStep): + def __str__(self): return "Migrating data" + from_level = "2.1.3" + to_level = "2.2-01" + def run(self): self.context.run_sql_resource("data-21to22.sql") + +class From22dataTo22(cloud_utils.MigrationStep): + def __str__(self): return "Migrating indexes" + from_level = "2.2-01" + to_level = "2.2" + def run(self): self.context.run_sql_resource("index-21to22.sql") + # command line harness functions def setup_logging(level): From b6322464ca6953dcd42e58c4285913bc79a9b2f9 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 11:30:32 -0700 Subject: [PATCH 04/11] Write the current schema level out to the database when doing deploydb or setup-databases --- setup/bindir/cloud-setup-databases.in | 7 +++++++ setup/db/schema-level.sql | 1 + wscript | 6 ++++++ 3 files changed, 14 insertions(+) create mode 100644 setup/db/schema-level.sql diff --git a/setup/bindir/cloud-setup-databases.in b/setup/bindir/cloud-setup-databases.in index 6c34575fcc0..3fe66d9c111 100755 --- a/setup/bindir/cloud-setup-databases.in +++ b/setup/bindir/cloud-setup-databases.in @@ -352,3 +352,10 @@ if rootuser: print "Applying file %s to the database on server %s:%s"%(p,host,port) try: run_mysql(text,rootuser,rootpassword,host,port) except CalledProcessError: sys.exit(22) + + p = os.path.join(dbfilepath,"schema-level.sql") + if os.path.isfile(p): + text = file(p).read() + print "Applying file %s to the database on server %s:%s"%(p,host,port) + try: run_mysql(text,rootuser,rootpassword,host,port) + except CalledProcessError: sys.exit(22) diff --git a/setup/db/schema-level.sql b/setup/db/schema-level.sql new file mode 100644 index 00000000000..4f2f0645482 --- /dev/null +++ b/setup/db/schema-level.sql @@ -0,0 +1 @@ +INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) VALUES ('Hidden', 'DEFAULT', 'database', 'schema.level', '2.1.3', 'The schema level of this database'); diff --git a/wscript b/wscript index 9286ded759a..e04ee7e8980 100644 --- a/wscript +++ b/wscript @@ -777,6 +777,12 @@ def deploydb(ctx,virttech=None): after = after + file(p).read() Utils.pprint("GREEN","Reading database code from %s"%p) + p = _join("setup","db","schema-level.sql") + if _exists(p): + p = dev_override(p) + after = after + file(p).read() + Utils.pprint("GREEN","Reading database code from %s"%p) + cmd = ["mysql","--user=%s"%dbuser,"-h",dbhost,"--password=%s"%dbpw] Utils.pprint("GREEN","Deploying post-configuration database scripts to %s (user %s)"%(dbhost,dbuser)) Utils.pprint("BLUE"," ".join(cmd)) From 89f6cb81f5fe12262a0bfb007b47c10bc6895a61 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 11:32:21 -0700 Subject: [PATCH 05/11] The database schema level for this branch is 2.2 --- setup/db/schema-level.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/db/schema-level.sql b/setup/db/schema-level.sql index 4f2f0645482..e3b0eea48fb 100644 --- a/setup/db/schema-level.sql +++ b/setup/db/schema-level.sql @@ -1 +1 @@ -INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) VALUES ('Hidden', 'DEFAULT', 'database', 'schema.level', '2.1.3', 'The schema level of this database'); +INSERT INTO `cloud`.`configuration` (category, instance, component, name, value, description) VALUES ('Hidden', 'DEFAULT', 'database', 'schema.level', '2.2', 'The schema level of this database'); From ef8e63c22cbcfb055f22663475003819060678c7 Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 11:48:42 -0700 Subject: [PATCH 06/11] Added database migration information in the handbook --- README.html | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.html b/README.html index 2ece7a070e7..192e2dd79cd 100644 --- a/README.html +++ b/README.html @@ -743,6 +743,28 @@ See the files in the {{{debian/}}} folder. 
Not done yet!
+
+
To support incremental migration from one version to another without having to redeploy the database, the CloudStack supports an incremental schema migration mechanism for the database.
+!!!How does it work?
+When the database is deployed for the first time with [[waf deploydb]] or the command {{{cloud-setup-databases}}}, a row is written to the {{{configuration}}} table, named {{{schema.level}}} and containing the current schema level.  This schema level row comes from the file {{{setup/db/schema-level.sql}}} in the source (refer to the [[Installation paths]] topic to find out where this file is installed in a running system).
+
+This value is used by the database migrator {{{cloud-migrate-databases}}} (source {{{setup/bindir/cloud-migrate-databases.in}}}) to determine the starting schema level.  The database migrator has a series of classes -- each class represents a step in the migration process and is usually tied to the execution of a SQL file stored in {{{setup/db}}}.  To migrate the database, the database migrator:
+# walks the list of steps it knows about,
+# generates a list of steps sorted by the order they should be executed in,
+# executes each step in order
+# at the end of each step, records the new schema level to the database table {{{configuration}}}
+For more information, refer to the database migrator source -- it is documented.
+!!!What impact does this have on me as a developer?
+Whenever you need to evolve the schema of the database:
+# write a migration SQL script and store it in {{{setup/db}}},
+# include your schema changes in the appropriate SQL file {{{create-*.sql}}} too (as the database is expected to be at its latest evolved schema level right after deploying a fresh database)
+# write a class in {{{setup/bindir/cloud-migrate-databases.in}}}, describing the migration step; in detail:
+## the schema level your migration step expects the database to be in,
+## the schema level your migration step will leave your database in (presumably the latest schema level, which you will have to choose!),
+## and the name / description of the step
+# bump the schema level in {{{setup/db/schema-level.sql}}} to the latest schema level
+Otherwise, ''end-user migration will fail catastrophically''.
+
[[Welcome]]
@@ -752,7 +774,7 @@ See the files in the {{{debian/}}} folder. 
Not done yet!
-
+
Start here if you want to learn the essentials to extend, modify and enhance the CloudStack.  This assumes that you've already familiarized yourself with CloudStack concepts, installation and configuration using the [[Getting started|Welcome]] instructions.
 * [[Obtain the source|Obtaining the source]]
 * [[Prepare your environment|Preparing your development environment]]
@@ -764,6 +786,7 @@ Extra developer information:
 * [[How to integrate with Eclipse]]
 * [[Starting over]]
 * [[Making a source release|waf dist]]
+* [[How to write database migration scripts|Database migration infrastructure]]
From f3aa8eccfa00516df06e427a5e2057615686028f Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 11:57:25 -0700 Subject: [PATCH 07/11] Add schema-level to the package manifests --- cloud.spec | 1 + debian/cloud-setup.install | 1 + 2 files changed, 2 insertions(+) diff --git a/cloud.spec b/cloud.spec index 690c816876e..addbb87fd63 100644 --- a/cloud.spec +++ b/cloud.spec @@ -582,6 +582,7 @@ fi %{_datadir}/%{name}/setup/index-212to213.sql %{_datadir}/%{name}/setup/postprocess-20to21.sql %{_datadir}/%{name}/setup/schema-20to21.sql +%{_datadir}/%{name}/setup/schema-level.sql %doc README %doc INSTALL %doc HACKING diff --git a/debian/cloud-setup.install b/debian/cloud-setup.install index 7d35dbe9929..4b7929819d1 100644 --- a/debian/cloud-setup.install +++ b/debian/cloud-setup.install @@ -12,3 +12,4 @@ /usr/share/cloud/setup/index-212to213.sql /usr/share/cloud/setup/postprocess-20to21.sql /usr/share/cloud/setup/schema-20to21.sql +/usr/share/cloud/setup/schema-level.sql From e7d4265500fe9dd27863bdac400fa9dfc2db118e Mon Sep 17 00:00:00 2001 From: "Manuel Amador (Rudd-O)" Date: Wed, 1 Sep 2010 11:59:21 -0700 Subject: [PATCH 08/11] Add 2.1 to 2.2 schema migration files to the package manifests --- cloud.spec | 2 ++ debian/cloud-setup.install | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cloud.spec b/cloud.spec index addbb87fd63..8207e103408 100644 --- a/cloud.spec +++ b/cloud.spec @@ -583,6 +583,8 @@ fi %{_datadir}/%{name}/setup/postprocess-20to21.sql %{_datadir}/%{name}/setup/schema-20to21.sql %{_datadir}/%{name}/setup/schema-level.sql +%{_datadir}/%{name}/setup/schema-21to22.sql +%{_datadir}/%{name}/setup/data-21to22.sql %doc README %doc INSTALL %doc HACKING diff --git a/debian/cloud-setup.install b/debian/cloud-setup.install index 4b7929819d1..48969370521 100644 --- a/debian/cloud-setup.install +++ b/debian/cloud-setup.install @@ -13,3 +13,5 @@ /usr/share/cloud/setup/postprocess-20to21.sql /usr/share/cloud/setup/schema-20to21.sql /usr/share/cloud/setup/schema-level.sql +/usr/share/cloud/setup/schema-21to22.sql +/usr/share/cloud/setup/data-21to22.sql From 398e5774aefc5d065adab044f209cfa224fa94cc Mon Sep 17 00:00:00 2001 From: abhishek Date: Wed, 1 Sep 2010 13:52:48 -0700 Subject: [PATCH 09/11] Merging 6065 into master --- server/src/com/cloud/server/ManagementServerImpl.java | 3 +++ server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index a1ce58b1599..64b42e24412 100644 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -921,6 +921,9 @@ public class ManagementServerImpl implements ManagementServer { // Mark the account's volumes as destroyed List volumes = _volumeDao.findDetachedByAccount(accountId); for (VolumeVO volume : volumes) { + if(volume.getPoolId()==null){ + accountCleanupNeeded = true; + } _storageMgr.destroyVolume(volume); } diff --git a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java index c6f82d2f567..d2cd55e2c68 100644 --- a/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java +++ b/server/src/com/cloud/storage/snapshot/SnapshotManagerImpl.java @@ -1006,6 +1006,9 @@ public class SnapshotManagerImpl implements SnapshotManager { // i.e Call them before the VMs for those volumes are destroyed. boolean success = true; for (VolumeVO volume : volumes) { + if(volume.getPoolId()==null){ + continue; + } Long volumeId = volume.getId(); Long dcId = volume.getDataCenterId(); String secondaryStoragePoolURL = _storageMgr.getSecondaryStorageURL(dcId); From 6cc9e740e4f4a23e0d27f9ac0e1ca573de808edd Mon Sep 17 00:00:00 2001 From: abhishek Date: Wed, 1 Sep 2010 14:06:35 -0700 Subject: [PATCH 10/11] organizing imports --- .../cloud/server/ManagementServerImpl.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index 64b42e24412..053bd0484a4 100644 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -101,15 +101,15 @@ import com.cloud.async.executor.SecurityGroupParam; import com.cloud.async.executor.UpdateLoadBalancerParam; import com.cloud.async.executor.UpgradeVMParam; import com.cloud.async.executor.VMOperationParam; -import com.cloud.async.executor.VMOperationParam.VmOp; import com.cloud.async.executor.VolumeOperationParam; +import com.cloud.async.executor.VMOperationParam.VmOp; import com.cloud.async.executor.VolumeOperationParam.VolumeOp; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; import com.cloud.configuration.ConfigurationManager; import com.cloud.configuration.ConfigurationVO; -import com.cloud.configuration.ResourceCount.ResourceType; import com.cloud.configuration.ResourceLimitVO; +import com.cloud.configuration.ResourceCount.ResourceType; import com.cloud.configuration.dao.ConfigurationDao; import com.cloud.configuration.dao.ResourceLimitDao; import com.cloud.consoleproxy.ConsoleProxyManager; @@ -119,8 +119,8 @@ import com.cloud.dc.DataCenterIpAddressVO; import com.cloud.dc.DataCenterVO; import com.cloud.dc.HostPodVO; import com.cloud.dc.PodVlanMapVO; -import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.VlanVO; +import com.cloud.dc.Vlan.VlanType; import com.cloud.dc.dao.AccountVlanMapDao; import com.cloud.dc.dao.ClusterDao; import com.cloud.dc.dao.DataCenterDao; @@ -175,7 +175,6 @@ import com.cloud.network.security.NetworkGroupRulesVO; import com.cloud.network.security.NetworkGroupVO; import com.cloud.network.security.dao.NetworkGroupDao; import com.cloud.offering.NetworkOffering; -import com.cloud.offering.NetworkOffering.GuestIpType; import com.cloud.offering.ServiceOffering; import com.cloud.serializer.GsonHelper; import com.cloud.server.auth.UserAuthenticator; @@ -188,13 +187,10 @@ import com.cloud.storage.GuestOSCategoryVO; import com.cloud.storage.GuestOSVO; import com.cloud.storage.LaunchPermissionVO; import com.cloud.storage.Snapshot; -import com.cloud.storage.Snapshot.SnapshotType; import com.cloud.storage.SnapshotPolicyVO; import com.cloud.storage.SnapshotScheduleVO; import com.cloud.storage.SnapshotVO; import com.cloud.storage.Storage; -import com.cloud.storage.Storage.FileSystem; -import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolHostVO; import com.cloud.storage.StoragePoolVO; @@ -202,9 +198,12 @@ import com.cloud.storage.StorageStats; import com.cloud.storage.VMTemplateHostVO; import com.cloud.storage.VMTemplateStorageResourceAssoc; import com.cloud.storage.VMTemplateVO; -import com.cloud.storage.Volume.VolumeType; import com.cloud.storage.VolumeStats; import com.cloud.storage.VolumeVO; +import com.cloud.storage.Snapshot.SnapshotType; +import com.cloud.storage.Storage.FileSystem; +import com.cloud.storage.Storage.ImageFormat; +import com.cloud.storage.Volume.VolumeType; import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.DiskTemplateDao; import com.cloud.storage.dao.GuestOSCategoryDao; @@ -215,9 +214,9 @@ import com.cloud.storage.dao.SnapshotPolicyDao; import com.cloud.storage.dao.StoragePoolDao; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VMTemplateDao; -import com.cloud.storage.dao.VMTemplateDao.TemplateFilter; import com.cloud.storage.dao.VMTemplateHostDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.storage.dao.VMTemplateDao.TemplateFilter; import com.cloud.storage.preallocatedlun.PreallocatedLunVO; import com.cloud.storage.preallocatedlun.dao.PreallocatedLunDao; import com.cloud.storage.secondary.SecondaryStorageVmManager; @@ -239,12 +238,12 @@ import com.cloud.user.dao.UserDao; import com.cloud.user.dao.UserStatisticsDao; import com.cloud.uservm.UserVm; import com.cloud.utils.DateUtil; -import com.cloud.utils.DateUtil.IntervalType; import com.cloud.utils.EnumUtils; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.PasswordGenerator; import com.cloud.utils.StringUtils; +import com.cloud.utils.DateUtil.IntervalType; import com.cloud.utils.component.Adapters; import com.cloud.utils.component.ComponentLocator; import com.cloud.utils.concurrency.NamedThreadFactory; From 1bbd8943e242372825b621fb28a792c6c92b6dde Mon Sep 17 00:00:00 2001 From: Jessica Wang Date: Wed, 1 Sep 2010 15:37:05 -0700 Subject: [PATCH 11/11] encode password before hashing it. --- ui/test/scripts/cloud.core.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/test/scripts/cloud.core.test.js b/ui/test/scripts/cloud.core.test.js index e9c61562647..0aacfd85a66 100644 --- a/ui/test/scripts/cloud.core.test.js +++ b/ui/test/scripts/cloud.core.test.js @@ -290,7 +290,7 @@ $(document).ready(function() { submenuContent.find("#grid_content").prepend(template.fadeIn("slow")); var username = thisDialog.find("#add_user_username").val(); - var password = $.md5(thisDialog.find("#add_user_password").val()); + var password = $.md5(encodeURIComponent(thisDialog.find("#add_user_password").val())); var email = thisDialog.find("#add_user_email").val(); if(email == "") email = username; @@ -318,7 +318,7 @@ $(document).ready(function() { $.ajax({ type: "POST", - data: createURL("command=createUser&username="+encodeURIComponent(username)+"&password="+encodeURIComponent(password)+"&email="+encodeURIComponent(email)+"&firstname="+encodeURIComponent(firstname)+"&lastname="+encodeURIComponent(lastname)+"&account="+account+"&accounttype="+accountType+"&domainid="+domainId+moreCriteria.join("")+"&response=json"), + data: createURL("command=createUser&username="+encodeURIComponent(username)+"&password="+password+"&email="+encodeURIComponent(email)+"&firstname="+encodeURIComponent(firstname)+"&lastname="+encodeURIComponent(lastname)+"&account="+account+"&accounttype="+accountType+"&domainid="+domainId+moreCriteria.join("")+"&response=json"), dataType: "json", async: false, success: function(json) {