CLOUDSTACK-8433: remove awsapi db usage and add upgrade cleanup path

- Removes awsapi db properties usage across codebase
- Removes references from spring xmls, test cases and TransactionLegacy
- Adds sql command to drop database cloudbridge in schema-451to460-cleanup.sql

Signed-off-by: Rohit Yadav <rohit.yadav@shapeblue.com>

.gitignore

@@ -37,7 +37,6 @@ build.number
 cloud.log.*.*
 unittest
 deps/cloud.userlibraries
-deps/awsapi-lib/
 .DS_Store
 .idea
 *.iml
@@ -61,14 +60,12 @@ tools/cli/build/
 #.*
 
 target-eclipse
-awsapi/modules/*
 !.gitignore
 .classpath
 .settings.xml
 .settings/
 db.properties.override
 replace.properties.override
-awsapi/overlays/
 tools/marvin/marvin/cloudstackAPI/*
 *.egg-info/
 docs/tmp

README.md

@@ -148,5 +148,4 @@ The following provides more details on the included cryptographic software:
 * CloudStack has a system requirement of MySQL, and uses native database encryption functionality.
 * CloudStack makes use of the Bouncy Castle general-purpose encryption library.
 * CloudStack can optionally interacts with and controls OpenSwan-based VPNs.
-* CloudStack has a dependency on Apache WSS4J as part of the AWSAPI implementation.
 * CloudStack has a dependency on and makes use of JSch - a java SSH2 implementation.

engine/storage/snapshot/test/resources/db.properties

@@ -55,9 +55,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

framework/db/src/com/cloud/utils/db/TransactionLegacy.java

@@ -77,7 +77,6 @@ public class TransactionLegacy implements Closeable {
 
     public static final short CLOUD_DB = 0;
     public static final short USAGE_DB = 1;
-    public static final short AWSAPI_DB = 2;
     public static final short SIMULATOR_DB = 3;
 
     public static final short CONNECTED_DB = -1;
@@ -229,19 +228,6 @@ public class TransactionLegacy implements Closeable {
         }
     }
 
-    public static Connection getStandaloneAwsapiConnection() {
-        try {
-            Connection conn = s_awsapiDS.getConnection();
-            if (s_connLogger.isTraceEnabled()) {
-                s_connLogger.trace("Retrieving a standalone connection for usage: dbconn" + System.identityHashCode(conn));
-            }
-            return conn;
-        } catch (SQLException e) {
-            s_logger.warn("Unexpected exception: ", e);
-            return null;
-        }
-    }
-
     public static Connection getStandaloneSimulatorConnection() {
         try {
             Connection conn = s_simulatorDS.getConnection();
@@ -571,15 +557,6 @@ public class TransactionLegacy implements Closeable {
                     throw new CloudRuntimeException("Database is not initialized, process is dying?");
                 }
                 break;
-            case AWSAPI_DB:
-                if (s_awsapiDS != null) {
-                    _conn = s_awsapiDS.getConnection();
-                } else {
-                    s_logger.warn("A static-initialized variable becomes null, process is dying?");
-                    throw new CloudRuntimeException("Database is not initialized, process is dying?");
-                }
-                break;
-
             case SIMULATOR_DB:
                 if (s_simulatorDS != null) {
                     _conn = s_simulatorDS.getConnection();
@@ -1014,7 +991,6 @@ public class TransactionLegacy implements Closeable {
 
     private static DataSource s_ds;
     private static DataSource s_usageDS;
-    private static DataSource s_awsapiDS;
     private static DataSource s_simulatorDS;
     private static boolean s_dbHAEnabled;
 
@@ -1136,20 +1112,6 @@ public class TransactionLegacy implements Closeable {
             // Data Source for usage server
             s_usageDS = new PoolingDataSource(usagePoolableConnectionFactory.getPool());
 
-            // Configure awsapi db
-            final String awsapiDbName = dbProps.getProperty("db.awsapi.name");
-            final GenericObjectPool awsapiConnectionPool =
-                    new GenericObjectPool(null, usageMaxActive, GenericObjectPool.DEFAULT_WHEN_EXHAUSTED_ACTION, usageMaxWait, usageMaxIdle);
-            final ConnectionFactory awsapiConnectionFactory =
-                    new DriverManagerConnectionFactory("jdbc:mysql://" + cloudHost + (s_dbHAEnabled ? "," + cloudSlaves : "") + ":" + cloudPort + "/" + awsapiDbName +
-                            "?autoReconnect=" + cloudAutoReconnect + (s_dbHAEnabled ? "&" + cloudDbHAParams : "") +
-                            (s_dbHAEnabled ? "&loadBalanceStrategy=" + loadBalanceStrategy : ""), cloudUsername, cloudPassword);
-            final PoolableConnectionFactory awsapiPoolableConnectionFactory =
-                    new PoolableConnectionFactory(awsapiConnectionFactory, awsapiConnectionPool, new StackKeyedObjectPoolFactory(), null, false, false);
-
-            // Data Source for awsapi
-            s_awsapiDS = new PoolingDataSource(awsapiPoolableConnectionFactory.getPool());
-
             try {
                 // Configure the simulator db
                 final int simulatorMaxActive = Integer.parseInt(dbProps.getProperty("db.simulator.maxActive"));

framework/db/test/com/cloud/utils/db/TestTransaction.java

@@ -127,30 +127,4 @@ public class TestTransaction {
         verify(conn, times(0)).rollback();
         verify(conn, times(1)).close();
     }
-
-    @Test
-    public void testOtherdatabaseRollback() throws Exception {
-        after();
-        setup(TransactionLegacy.AWSAPI_DB);
-
-        try {
-            Transaction.execute(new TransactionCallbackNoReturn() {
-                @Override
-                public void doInTransactionWithoutResult(TransactionStatus status) {
-                    assertEquals(TransactionLegacy.AWSAPI_DB, TransactionLegacy.currentTxn().getDatabaseId().shortValue());
-
-                    throw new RuntimeException("Panic!");
-                }
-            });
-            fail();
-        } catch (RuntimeException e) {
-            assertEquals("Panic!", e.getMessage());
-        }
-
-        verify(conn).setAutoCommit(false);
-        verify(conn, times(0)).commit();
-        verify(conn, times(1)).rollback();
-        verify(conn, times(1)).close();
-    }
-
 }

framework/db/test/db.properties

@@ -58,13 +58,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.username=cloud
-db.awsapi.password=cloud
-db.awsapi.host=localhost
-db.awsapi.port=3306
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

framework/jobs/test/resources/db.properties

@@ -51,9 +51,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

plugins/network-elements/dns-notifier/resources/components-example.xml

@@ -188,31 +188,4 @@ under the License.
         <dao name="Site2SiteVpnGatewayDao" class="com.cloud.network.dao.Site2SiteVpnGatewayDaoImpl" singleton="false"/>
         <dao name="Site2SiteVpnConnectionDao" class="com.cloud.network.dao.Site2SiteVpnConnectionDaoImpl" singleton="false"/>
     </configuration-server>
-    
-    <awsapi-ec2server class="com.cloud.bridge.service.EC2MainServlet">
-        <dao name="CloudStackConfigurationDao" class="com.cloud.bridge.persist.dao.CloudStackConfigurationDaoImpl" singleton="false"/>
-        <dao name="UserCredentialsDao" class="com.cloud.bridge.persist.dao.UserCredentialsDaoImpl" singleton="false"/>
-        <dao name="CloudStackSvcOfferingDao" class="com.cloud.bridge.persist.dao.CloudStackSvcOfferingDaoImpl" singleton="false"/>
-        <dao name="OfferingDao" class="com.cloud.bridge.persist.dao.OfferingDaoImpl" singleton="false"/>
-        <dao name="CloudStackAccountDao" class="com.cloud.bridge.persist.dao.CloudStackAccountDaoImpl" singleton="false"/>
-    </awsapi-ec2server>
-    
-    <awsapi-s3server class="com.cloud.bridge.service.S3RestServlet">
-        <dao name="CloudStackConfigurationDao" class="com.cloud.bridge.persist.dao.CloudStackConfigurationDaoImpl" singleton="false"/>
-        <dao name="MHostDao" class="com.cloud.bridge.persist.dao.MHostDaoImpl" singleton="false"/>
-        <dao name="SHostDao" class="com.cloud.bridge.persist.dao.SHostDaoImpl" singleton="false"/>
-        <dao name="UserCredentialsDao" class="com.cloud.bridge.persist.dao.UserCredentialsDaoImpl" singleton="false"/>
-        <dao name="BucketPolicyDao" class="com.cloud.bridge.persist.dao.BucketPolicyDaoImpl" singleton="false"/>
-        <dao name="MHostMountDao" class="com.cloud.bridge.persist.dao.MHostMountDaoImpl" singleton="false"/>
-        <dao name="SAclDao" class="com.cloud.bridge.persist.dao.SAclDaoImpl" singleton="false"/>
-        <dao name="SBucketDao" class="com.cloud.bridge.persist.dao.SBucketDaoImpl" singleton="false"/>
-        <dao name="SMetaDao" class="com.cloud.bridge.persist.dao.SMetaDaoImpl" singleton="false"/>
-        <dao name="SObjectDao" class="com.cloud.bridge.persist.dao.SObjectDaoImpl" singleton="false"/>
-        <dao name="SObjectItemDao" class="com.cloud.bridge.persist.dao.SObjectItemDaoImpl" singleton="false"/>
-        <dao name="MultiPartPartsDao" class="com.cloud.bridge.persist.dao.MultiPartPartsDaoImpl" singleton="false"/>
-        <dao name="MultiPartUploadsDao" class="com.cloud.bridge.persist.dao.MultiPartUploadsDaoImpl" singleton="false"/>
-        <dao name="MultipartMetaDao" class="com.cloud.bridge.persist.dao.MultipartMetaDaoImpl" singleton="false"/>
-        <dao name="UserCredentialsDao" class="com.cloud.bridge.persist.dao.UserCredentialsDaoImpl" singleton="false"/>
-    </awsapi-s3server>
-    
 </components.xml>

plugins/network-elements/globodns/test/resources/db.properties

@@ -56,13 +56,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.username=cloud
-db.awsapi.password=cloud
-db.awsapi.host=localhost
-db.awsapi.port=3306
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

plugins/network-elements/juniper-contrail/test/org/apache/cloudstack/network/contrail/management/TestDbSetup.java

@@ -94,10 +94,6 @@ public class TestDbSetup {
         config.setProperty("db.usage.username", System.getProperty("user.name"));
         config.setProperty("db.usage.password", "");
 
-        config.setProperty("db.awsapi.port", "" + port);
-        config.setProperty("db.awsapi.username", System.getProperty("user.name"));
-        config.setProperty("db.awsapi.password", "");
-
         config.setProperty("db.simulator.port", "" + port);
         config.setProperty("db.simulator.username", System.getProperty("user.name"));
         config.setProperty("db.simulator.password", "");

plugins/network-elements/juniper-contrail/test/resources/db.properties

@@ -51,9 +51,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

server/test/resources/db.properties

@@ -55,9 +55,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

services/iam/plugin/test/resources/db.properties

@@ -56,13 +56,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.username=cloud
-db.awsapi.password=cloud
-db.awsapi.host=localhost
-db.awsapi.port=3306
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

services/iam/server/test/resources/db.properties

@@ -56,13 +56,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.username=cloud
-db.awsapi.password=cloud
-db.awsapi.host=localhost
-db.awsapi.port=3306
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

setup/bindir/cloud-setup-databases.in

@@ -265,26 +265,6 @@ for full help
             self.runMysql(text, p, True)
             self.info(None, True)
 
-        awsApiDbDir = '/usr/share/cloudstack-bridge/setup'
-        for f in ["cloudbridge_db.sql"]:
-            p = os.path.join(awsApiDbDir,f)
-            if not os.path.exists(p): continue
-            text = file(p).read()
-            for t, r in replacements: text = text.replace(t,r)
-            self.info("Applying %s"%p)
-            self.runMysql(text, p, True)
-            self.info(None, True)
-
-        for f in ["cloudbridge_schema", "cloudbridge_multipart", "cloudbridge_index", "cloudbridge_multipart_alter", "cloudbridge_bucketpolicy", "cloudbridge_policy_alter",
-            "cloudbridge_offering", "cloudbridge_offering_alter"]:
-            if os.path.isfile(p):
-                p = os.path.join(awsApiDbDir,"%s.sql"%f)
-                if not os.path.exists(p): continue
-                text = file(p).read()
-                self.info("Applying %s"%p)
-                self.runMysql(text, p, True)
-                self.info(None, True)
-
     def prepareDBFiles(self):
         def prepareDBDotProperties():
             dbpPath = os.path.join(self.dbConfPath, 'db.properties')

setup/db/db/schema-451to460-cleanup.sql

@@ -18,3 +18,5 @@
 --
 -- Schema cleanup from 4.5.1 to 4.6.0
 --
+
+DROP DATABASE IF EXISTS `cloudbridge`;

tools/eclipse/eclipse.epf

@@ -428,7 +428,7 @@
 /instance/org.eclipse.jdt.core/org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
 /instance/org.eclipse.jdt.core/org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
 /instance/org.eclipse.jdt.core/org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-/instance/org.eclipse.wst.sse.core/task-tag-projects-already-scanned=cloud-agent,cloud-api,cloud-apidoc,cloud-awsapi,cloud-cli,cloud-client-ui,cloud-console-proxy,cloud-core,cloud-devcloud,cloud-devcloud-kvm,cloud-developer,cloud-engine,cloud-engine-api,cloud-engine-components-api,cloud-engine-compute,cloud-engine-network,cloud-engine-orchestration,cloud-engine-schema,cloud-engine-service,cloud-engine-storage,cloud-engine-storage-backup,cloud-engine-storage-image,cloud-engine-storage-imagemotion,cloud-engine-storage-integration-test,cloud-engine-storage-snapshot,cloud-engine-storage-volume,cloud-framework-events,cloud-framework-ipc,cloud-framework-rest,cloud-marvin,cloud-mom-rabbitmq,cloud-patches,cloud-plugin-acl-static-role-based,cloud-plugin-api-discovery,cloud-plugin-api-limit-account-based,cloud-plugin-console-proxy,cloud-plugin-example-dns-notifier,cloud-plugin-host-allocator-random,cloud-plugin-hypervisor-baremetal,cloud-plugin-hypervisor-kvm,cloud-plugin-hypervisor-ovm,cloud-plugin-hypervisor-simulator,cloud-plugin-hypervisor-ucs,cloud-plugin-hypervisor-vmware,cloud-plugin-hypervisor-xen,cloud-plugin-netapp,cloud-plugin-network-elb,cloud-plugin-network-f5,cloud-plugin-network-netscaler,cloud-plugin-network-nvp,cloud-plugin-network-ovs,cloud-plugin-network-srx,cloud-plugin-network-bigswitch,cloud-plugin-planner-user-concentrated-pod,cloud-plugin-planner-user-dispersing,cloud-plugin-snmp-alerts,cloud-plugin-storage-allocator-random,cloud-plugin-storage-image-s3,cloud-plugin-storage-volume-default,cloud-plugin-storage-volume-solidfire,cloud-plugin-syslog-alerts,cloud-plugin-user-authenticator-ldap,cloud-plugin-user-authenticator-md5,cloud-plugin-user-authenticator-plaintext,cloud-plugin-user-authenticator-saml2,cloud-plugin-user-authenticator-sha256salted,cloud-server,cloud-testclient,cloud-tools,cloud-usage,cloud-utils,cloud-vmware-base,cloudstack,cloudstack-framework,cloudstack-plugins,xapi,cloud-framework-jobs,cloud-plugin-host-anti-affinity,cloud-plugin-network-cisco-vnmc,cloud-plugin-network-midonet,cloud-secondary-storage,cloudstack-services,cloud-plugin-network-internallb,cloud-plugin-planner-implicit-dedication,cloudstack-service-console-proxy
+/instance/org.eclipse.wst.sse.core/task-tag-projects-already-scanned=cloud-agent,cloud-api,cloud-apidoc,cloud-cli,cloud-client-ui,cloud-console-proxy,cloud-core,cloud-devcloud,cloud-devcloud-kvm,cloud-developer,cloud-engine,cloud-engine-api,cloud-engine-components-api,cloud-engine-compute,cloud-engine-network,cloud-engine-orchestration,cloud-engine-schema,cloud-engine-service,cloud-engine-storage,cloud-engine-storage-backup,cloud-engine-storage-image,cloud-engine-storage-imagemotion,cloud-engine-storage-integration-test,cloud-engine-storage-snapshot,cloud-engine-storage-volume,cloud-framework-events,cloud-framework-ipc,cloud-framework-rest,cloud-marvin,cloud-mom-rabbitmq,cloud-patches,cloud-plugin-acl-static-role-based,cloud-plugin-api-discovery,cloud-plugin-api-limit-account-based,cloud-plugin-console-proxy,cloud-plugin-example-dns-notifier,cloud-plugin-host-allocator-random,cloud-plugin-hypervisor-baremetal,cloud-plugin-hypervisor-kvm,cloud-plugin-hypervisor-ovm,cloud-plugin-hypervisor-simulator,cloud-plugin-hypervisor-ucs,cloud-plugin-hypervisor-vmware,cloud-plugin-hypervisor-xen,cloud-plugin-netapp,cloud-plugin-network-elb,cloud-plugin-network-f5,cloud-plugin-network-netscaler,cloud-plugin-network-nvp,cloud-plugin-network-ovs,cloud-plugin-network-srx,cloud-plugin-network-bigswitch,cloud-plugin-planner-user-concentrated-pod,cloud-plugin-planner-user-dispersing,cloud-plugin-snmp-alerts,cloud-plugin-storage-allocator-random,cloud-plugin-storage-image-s3,cloud-plugin-storage-volume-default,cloud-plugin-storage-volume-solidfire,cloud-plugin-syslog-alerts,cloud-plugin-user-authenticator-ldap,cloud-plugin-user-authenticator-md5,cloud-plugin-user-authenticator-plaintext,cloud-plugin-user-authenticator-saml2,cloud-plugin-user-authenticator-sha256salted,cloud-server,cloud-testclient,cloud-tools,cloud-usage,cloud-utils,cloud-vmware-base,cloudstack,cloudstack-framework,cloudstack-plugins,xapi,cloud-framework-jobs,cloud-plugin-host-anti-affinity,cloud-plugin-network-cisco-vnmc,cloud-plugin-network-midonet,cloud-secondary-storage,cloudstack-services,cloud-plugin-network-internallb,cloud-plugin-planner-implicit-dedication,cloudstack-service-console-proxy
 /instance/org.eclipse.jdt.debug.ui/org.eclipse.debug.ui.ExpressionView.org.eclipse.jdt.debug.ui.show_null_entries=true
 /instance/org.eclipse.jdt.ui/sp_cleanup.correct_indentation=false
 /instance/org.eclipse.jdt.core/org.eclipse.jdt.core.formatter.comment.format_source_code=true

usage/test/resources/db.properties

@@ -55,9 +55,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud

utils/conf/db.properties

@@ -58,13 +58,6 @@ db.usage.maxIdle=30
 db.usage.maxWait=10000
 db.usage.autoReconnect=true
 
-# awsapi database settings
-db.awsapi.username=cloud
-db.awsapi.password=cloud
-db.awsapi.host=localhost
-db.awsapi.port=3306
-db.awsapi.name=cloudbridge
-
 # Simulator database settings
 db.simulator.username=cloud
 db.simulator.password=cloud