From 6be228a438e5ce3fba831be8130c73603cb0457b Mon Sep 17 00:00:00 2001 From: Min Chen Date: Mon, 28 Oct 2013 21:01:31 -0700 Subject: [PATCH] CLOUDSTACK-4024:Provide a way to upgrade from existing NFS secondary storage to S3. --- api/src/com/cloud/event/EventTypes.java | 3 + api/src/com/cloud/storage/StorageService.java | 16 +- .../cloudstack/api/ApiCommandJobType.java | 1 + ...repareSecondaryStorageForMigrationCmd.java | 109 ++++++++ client/tomcatconf/commands.properties.in | 1 + .../api/storage/DataStoreLifeCycle.java | 2 + .../api/storage/DataStoreManager.java | 3 + .../api/storage/SnapshotDataFactory.java | 4 + .../api/storage/TemplateDataFactory.java | 6 + .../api/storage/TemplateService.java | 6 +- .../api/storage/VolumeDataFactory.java | 4 + .../storage/datastore/db/ImageStoreDao.java | 2 + .../datastore/db/SnapshotDataStoreDao.java | 9 + .../datastore/db/SnapshotDataStoreVO.java | 34 ++- .../datastore/db/TemplateDataStoreDao.java | 10 +- .../datastore/db/TemplateDataStoreVO.java | 51 ++-- .../datastore/db/VolumeDataStoreDao.java | 2 + .../datastore/db/VolumeDataStoreVO.java | 47 ++-- .../motion/AncientDataMotionStrategy.java | 37 ++- .../image/TemplateDataFactoryImpl.java | 38 ++- .../storage/image/TemplateServiceImpl.java | 95 ++++++- .../ImageStoreProviderManagerImpl.java | 15 +- .../storage/image/store/TemplateObject.java | 8 + .../snapshot/SnapshotDataFactoryImpl.java | 22 +- .../snapshot/XenserverSnapshotStrategy.java | 57 ++-- .../datastore/DataStoreManagerImpl.java | 5 + .../image/datastore/ImageStoreHelper.java | 21 +- .../datastore/ImageStoreProviderManager.java | 2 + .../storage/image/db/ImageStoreDaoImpl.java | 10 +- .../image/db/SnapshotDataStoreDaoImpl.java | 82 +++++- .../image/db/TemplateDataStoreDaoImpl.java | 104 +++++++ .../image/db/VolumeDataStoreDaoImpl.java | 55 +++- .../storage/volume/VolumeDataFactoryImpl.java | 33 ++- .../SimulatorImageStoreLifeCycleImpl.java | 36 ++- .../CloudStackImageStoreLifeCycleImpl.java | 9 + .../lifecycle/S3ImageStoreLifeCycleImpl.java | 12 +- .../SampleImageStoreLifeCycleImpl.java | 9 + .../SwiftImageStoreLifeCycleImpl.java | 30 ++- ...oudStackPrimaryDataStoreLifeCycleImpl.java | 17 +- .../SamplePrimaryDataStoreLifeCycleImpl.java | 34 ++- .../SolidFirePrimaryDataStoreLifeCycle.java | 254 +++++++++--------- .../cloud/server/ManagementServerImpl.java | 71 ++--- .../com/cloud/storage/StorageManagerImpl.java | 74 +++-- .../cloud/storage/VolumeApiServiceImpl.java | 125 +++++---- .../template/HypervisorTemplateAdapter.java | 29 +- .../cloud/template/TemplateManagerImpl.java | 91 ++++--- tools/apidoc/gen_toc.py | 1 + 47 files changed, 1238 insertions(+), 448 deletions(-) create mode 100644 api/src/org/apache/cloudstack/api/command/admin/storage/PrepareSecondaryStorageForMigrationCmd.java diff --git a/api/src/com/cloud/event/EventTypes.java b/api/src/com/cloud/event/EventTypes.java index c7e7a45cfdd..d9f80eb2a2c 100755 --- a/api/src/com/cloud/event/EventTypes.java +++ b/api/src/com/cloud/event/EventTypes.java @@ -449,6 +449,9 @@ public class EventTypes { public static final String EVENT_UCS_ASSOCIATED_PROFILE = "UCS.ASSOCIATEPROFILE"; + // Object store migration + public static final String EVENT_MIGRATE_PREPARE_SECONDARY_STORAGE = "MIGRATE.PREPARE.SS"; + static { // TODO: need a way to force author adding event types to declare the entity details as well, with out braking diff --git a/api/src/com/cloud/storage/StorageService.java b/api/src/com/cloud/storage/StorageService.java index 1ae1d3a7102..cbbc1f33559 100644 --- a/api/src/com/cloud/storage/StorageService.java +++ b/api/src/com/cloud/storage/StorageService.java @@ -22,9 +22,9 @@ import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd; -import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import com.cloud.exception.DiscoveryException; @@ -97,4 +97,18 @@ public interface StorageService{ ImageStore discoverImageStore(AddImageStoreCmd cmd) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException; + /** + * Prepare NFS secondary storage for object store migration + * + * @param cmd + * - the command specifying secondaryStorageId + * @return the storage pool + * @throws ResourceUnavailableException + * TODO + * @throws InsufficientCapacityException + * TODO + */ + public ImageStore prepareSecondaryStorageForObjectStoreMigration(Long storeId) throws ResourceUnavailableException, + InsufficientCapacityException; + } diff --git a/api/src/org/apache/cloudstack/api/ApiCommandJobType.java b/api/src/org/apache/cloudstack/api/ApiCommandJobType.java index 204fe3ed8f4..3067af8a528 100644 --- a/api/src/org/apache/cloudstack/api/ApiCommandJobType.java +++ b/api/src/org/apache/cloudstack/api/ApiCommandJobType.java @@ -28,6 +28,7 @@ public enum ApiCommandJobType { SystemVm, Host, StoragePool, + ImageStore, IpAddress, PortableIpAddress, SecurityGroup, diff --git a/api/src/org/apache/cloudstack/api/command/admin/storage/PrepareSecondaryStorageForMigrationCmd.java b/api/src/org/apache/cloudstack/api/command/admin/storage/PrepareSecondaryStorageForMigrationCmd.java new file mode 100644 index 00000000000..d0c995a64f1 --- /dev/null +++ b/api/src/org/apache/cloudstack/api/command/admin/storage/PrepareSecondaryStorageForMigrationCmd.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.log4j.Logger; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandJobType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.context.CallContext; + +import com.cloud.event.EventTypes; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.storage.ImageStore; +import com.cloud.user.Account; + +@APICommand(name = "prepareSecondaryStorageForMigration", description = "Prepare a NFS secondary storage to migrate to use object store like S3", responseObject = ImageStoreResponse.class) +public class PrepareSecondaryStorageForMigrationCmd extends BaseAsyncCmd { + public static final Logger s_logger = Logger.getLogger(PrepareSecondaryStorageForMigrationCmd.class.getName()); + private static final String s_name = "preparesecondarystorageformigrationresponse"; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, + required = true, description = "Secondary image store ID") + private Long id; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public String getCommandName() { + return s_name; + } + + @Override + public ApiCommandJobType getInstanceType() { + return ApiCommandJobType.ImageStore; + } + + @Override + public Long getInstanceId() { + return getId(); + } + + @Override + public long getEntityOwnerId() { + Account account = CallContext.current().getCallingAccount(); + if (account != null) { + return account.getId(); + } + + return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked + } + + @Override + public String getEventType() { + return EventTypes.EVENT_MIGRATE_PREPARE_SECONDARY_STORAGE; + } + + @Override + public String getEventDescription() { + return "preparing secondary storage: " + getId() + " for object store migration"; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException{ + ImageStore result = _storageService.prepareSecondaryStorageForObjectStoreMigration(getId()); + if (result != null){ + ImageStoreResponse response = _responseGenerator.createImageStoreResponse(result); + response.setResponseName(getCommandName()); + response.setResponseName("secondarystorage"); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to prepare secondary storage for object store migration"); + } + } +} diff --git a/client/tomcatconf/commands.properties.in b/client/tomcatconf/commands.properties.in index 3fe046386df..1503ab853c1 100644 --- a/client/tomcatconf/commands.properties.in +++ b/client/tomcatconf/commands.properties.in @@ -256,6 +256,7 @@ deleteImageStore=1 createSecondaryStagingStore=1 listSecondaryStagingStores=1 deleteSecondaryStagingStore=1 +prepareSecondaryStorageForMigration=1 #### host commands addHost=3 diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreLifeCycle.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreLifeCycle.java index 1e893db6bb5..c881570ba12 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreLifeCycle.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreLifeCycle.java @@ -37,4 +37,6 @@ public interface DataStoreLifeCycle { boolean cancelMaintain(DataStore store); boolean deleteDataStore(DataStore store); + + boolean migrateToObjectStore(DataStore store); } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java index 08844536264..7fbec0ad35f 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.engine.subsystem.api.storage; import java.util.List; + import com.cloud.storage.DataStoreRole; public interface DataStoreManager { @@ -37,4 +38,6 @@ public interface DataStoreManager { DataStore getImageCacheStore(long zoneId); List listImageStores(); + + List listImageCacheStores(); } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java index 0b8d1f104e1..d5255f40407 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import java.util.List; + import com.cloud.storage.DataStoreRole; public interface SnapshotDataFactory { @@ -26,4 +28,6 @@ public interface SnapshotDataFactory { SnapshotInfo getSnapshot(DataObject obj, DataStore store); SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role); + + List listSnapshotOnCache(long snapshotId); } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java index 0b78da058ed..801c4427f50 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import java.util.List; + import com.cloud.storage.DataStoreRole; public interface TemplateDataFactory { @@ -28,4 +30,8 @@ public interface TemplateDataFactory { TemplateInfo getTemplate(long templateId, DataStoreRole storeRole); TemplateInfo getTemplate(long templateId, DataStoreRole storeRole, Long zoneId); + + TemplateInfo getReadyTemplateOnCache(long templateId); + + List listTemplateOnCache(long templateId); } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java index 4950597963d..185d2545eb7 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java @@ -36,7 +36,7 @@ public interface TemplateService { } public TemplateInfo getTemplate() { - return this.template; + return template; } } @@ -54,6 +54,8 @@ public interface TemplateService { AsyncCallFuture prepareTemplateOnPrimary(TemplateInfo srcTemplate, StoragePool pool); + void syncTemplateToRegionStore(long templateId, DataStore store); + void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId); void handleTemplateSync(DataStore store); @@ -62,5 +64,7 @@ public interface TemplateService { void addSystemVMTemplatesToSecondary(DataStore store); + void associateTemplateToZone(long templateId, Long zoneId); + void associateCrosszoneTemplatesToZone(long dcId); } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeDataFactory.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeDataFactory.java index 99e3b596071..3de0b5b4c73 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeDataFactory.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeDataFactory.java @@ -18,6 +18,8 @@ */ package org.apache.cloudstack.engine.subsystem.api.storage; +import java.util.List; + import com.cloud.storage.DataStoreRole; public interface VolumeDataFactory { @@ -28,4 +30,6 @@ public interface VolumeDataFactory { VolumeInfo getVolume(long volumeId, DataStoreRole storeRole); VolumeInfo getVolume(long volumeId); + + List listVolumeOnCache(long volumeId); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java index f95e66cd498..d0f8fe87850 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java @@ -36,4 +36,6 @@ public interface ImageStoreDao extends GenericDao { List findImageCacheByScope(ZoneScope scope); List listImageStores(); + + List listImageCacheStores(); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index e350a763449..dfa03adf29b 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -41,4 +41,13 @@ StateDao listDestroyed(long storeId); List findBySnapshotId(long snapshotId); + + void duplicateCacheRecordsOnRegionStore(long storeId); + + // delete the snapshot entry on primary data store to make sure that next snapshot will be full snapshot + void deleteSnapshotRecordsOnPrimary(); + + List listOnCache(long snapshotId); + + void updateStoreRoleToCache(long storeId); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java index 0fe5e088043..db86c3f24b7 100644 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java @@ -34,7 +34,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import com.cloud.storage.DataStoreRole; -import com.cloud.utils.db.GenericDao; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.fsm.StateObject; @@ -98,6 +97,7 @@ public class SnapshotDataStoreVO implements StateObject listByTemplate(long templateId); + + void duplicateCacheRecordsOnRegionStore(long storeId); + + TemplateDataStoreVO findReadyOnCache(long templateId); + + List listOnCache(long templateId); + + void updateStoreRoleToCachce(long storeId); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java index a890e4b337b..a3696d8d75f 100755 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java @@ -35,7 +35,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState import com.cloud.storage.DataStoreRole; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; - import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.fsm.StateObject; @@ -117,17 +116,17 @@ public class TemplateDataStoreVO implements StateObject listDestroyed(long storeId); + + void duplicateCacheRecordsOnRegionStore(long storeId); } diff --git a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java index e11071b702e..e34d4a60998 100755 --- a/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java +++ b/engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java @@ -33,8 +33,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; -import com.cloud.storage.Storage; -import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.fsm.StateObject; @@ -117,6 +115,7 @@ public class VolumeDataStoreVO implements StateObject listTemplateOnCache(long templateId) { + List cacheTmpls = templateStoreDao.listOnCache(templateId); + List tmplObjs = new ArrayList(); + for (TemplateDataStoreVO cacheTmpl : cacheTmpls) { + long storeId = cacheTmpl.getDataStoreId(); + DataStore store = storeMgr.getDataStore(storeId, DataStoreRole.ImageCache); + TemplateInfo tmplObj = getTemplate(templateId, store); + tmplObjs.add(tmplObj); + } + return tmplObjs; + } + } diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java index a68e40c63f3..ce6198dc21e 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java @@ -28,6 +28,9 @@ import java.util.Set; import javax.inject.Inject; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult; import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult; import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService; @@ -39,6 +42,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo; import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; @@ -59,9 +63,6 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.store.TemplateObject; import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - import com.cloud.agent.api.Answer; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; @@ -73,8 +74,9 @@ import com.cloud.dc.dao.DataCenterDao; import com.cloud.exception.ResourceAllocationException; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.StoragePool; +import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.TemplateType; +import com.cloud.storage.StoragePool; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; @@ -128,6 +130,8 @@ public class TemplateServiceImpl implements TemplateService { TemplateManager _tmpltMgr; @Inject ConfigurationDao _configDao; + @Inject + StorageCacheManager _cacheMgr; class TemplateOpContext extends AsyncRpcContext { final TemplateObject template; @@ -466,7 +470,8 @@ public class TemplateServiceImpl implements TemplateService { // persist entry in template_zone_ref table. zoneId can be empty for // region-wide image store, in that case, // we will associate the template to all the zones. - private void associateTemplateToZone(long templateId, Long zoneId) { + @Override + public void associateTemplateToZone(long templateId, Long zoneId) { List dcs = new ArrayList(); if (zoneId != null) { dcs.add(zoneId); @@ -608,6 +613,86 @@ public class TemplateServiceImpl implements TemplateService { return copyAsync(volume, template, store); } + private AsyncCallFuture syncToRegionStoreAsync(TemplateInfo template, DataStore store) { + AsyncCallFuture future = new AsyncCallFuture(); + // no need to create entry on template_store_ref here, since entries are already created when prepareSecondaryStorageForMigration is invoked. + // But we need to set default install path so that sync can be done in the right s3 path + TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store); + String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/" + + TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR + + template.getAccountId() + "/" + template.getId() + "/" + template.getUniqueName(); + ((TemplateObject)templateOnStore).setInstallPath(installPath); + TemplateOpContext context = new TemplateOpContext(null, + (TemplateObject)templateOnStore, future); + AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); + caller.setCallback(caller.getTarget().syncTemplateCallBack(null, null)).setContext(context); + _motionSrv.copyAsync(template, templateOnStore, caller); + return future; + } + + protected Void syncTemplateCallBack(AsyncCallbackDispatcher callback, + TemplateOpContext context) { + TemplateInfo destTemplate = context.getTemplate(); + CopyCommandResult result = callback.getResult(); + AsyncCallFuture future = context.getFuture(); + TemplateApiResult res = new TemplateApiResult(destTemplate); + try { + if (result.isFailed()) { + res.setResult(result.getResult()); + // no change to existing template_store_ref, will try to re-sync later if other call triggers this sync operation, like copy template + } else { + // this will update install path properly, next time it will not sync anymore. + destTemplate.processEvent(Event.OperationSuccessed, result.getAnswer()); + } + future.complete(res); + } catch (Exception e) { + s_logger.debug("Failed to process sync template callback", e); + res.setResult(e.toString()); + future.complete(res); + } + + return null; + } + + private boolean isRegionStore(DataStore store) { + if (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId() == null) + return true; + else + return false; + } + + // This routine is used to push templates currently on cache store, but not in region store to region store. + // used in migrating existing NFS secondary storage to S3. + @Override + public void syncTemplateToRegionStore(long templateId, DataStore store) { + if (isRegionStore(store)) { + // if template is on region wide object store, check if it is really downloaded there (by checking install_path). Sync template to region + // wide store if it is not there physically. + TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store); + if (tmplOnStore == null) { + throw new CloudRuntimeException("Cannot find an entry in template_store_ref for template " + templateId + " on region store: " + store.getName()); + } + if (tmplOnStore.getInstallPath() == null || tmplOnStore.getInstallPath().length() == 0) { + // template is not on region store yet, sync to region store + TemplateInfo srcTemplate = _templateFactory.getReadyTemplateOnCache(templateId); + if (srcTemplate == null) { + throw new CloudRuntimeException("Cannot find template " + templateId + " on cache store"); + } + AsyncCallFuture future = syncToRegionStoreAsync(srcTemplate, store); + try { + TemplateApiResult result = future.get(); + if (result.isFailed()) { + throw new CloudRuntimeException("sync template from cache to region wide store failed for image store " + store.getName() + ":" + + result.getResult()); + } + _cacheMgr.releaseCacheObject(srcTemplate); // reduce reference count for template on cache, so it can recycled by schedule + } catch (Exception ex) { + throw new CloudRuntimeException("sync template from cache to region wide store failed for image store " + store.getName()); + } + } + } + } + @Override public AsyncCallFuture copyTemplate(TemplateInfo srcTemplate, DataStore destStore) { // generate a URL from source template ssvm to download to destination data store diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java index 64ef78f8e09..0991860c4d5 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java @@ -26,6 +26,9 @@ import java.util.Map; import javax.annotation.PostConstruct; import javax.inject.Inject; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider; @@ -37,8 +40,6 @@ import org.apache.cloudstack.storage.image.ImageStoreDriver; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.ImageStoreImpl; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.storage.ScopeType; import com.cloud.storage.dao.VMTemplateDao; @@ -94,6 +95,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager return imageStores; } + @Override + public List listImageCacheStores() { + List stores = dataStoreDao.listImageCacheStores(); + List imageStores = new ArrayList(); + for (ImageStoreVO store : stores) { + imageStores.add(getImageStore(store.getId())); + } + return imageStores; + } + @Override public List listImageStoresByScope(ZoneScope scope) { List stores = dataStoreDao.findByScope(scope); diff --git a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java index f0675f3ee27..0a5b60880e5 100644 --- a/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java +++ b/engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java @@ -58,6 +58,7 @@ public class TemplateObject implements TemplateInfo { private VMTemplateVO imageVO; private DataStore dataStore; private String url; + private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case @Inject VMTemplateDao imageDao; @Inject @@ -293,6 +294,9 @@ public class TemplateObject implements TemplateInfo { @Override public String getInstallPath() { + if (installPath != null) + return installPath; + if (dataStore == null) { return null; } @@ -300,6 +304,10 @@ public class TemplateObject implements TemplateInfo { return obj.getInstallPath(); } + public void setInstallPath(String installPath) { + this.installPath = installPath; + } + @Override public long getAccountId() { return imageVO.getAccountId(); diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index aafdad05ff0..6205fe40deb 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -18,8 +18,13 @@ */ package org.apache.cloudstack.storage.snapshot; +import java.util.ArrayList; +import java.util.List; + import javax.inject.Inject; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -28,7 +33,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.springframework.stereotype.Component; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; @@ -70,8 +74,22 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory { if (snapshotStore == null) { return null; } - DataStore store = this.storeMgr.getDataStore(snapshotStore.getDataStoreId(), role); + DataStore store = storeMgr.getDataStore(snapshotStore.getDataStoreId(), role); SnapshotObject so = SnapshotObject.getSnapshotObject(snapshot, store); return so; } + + @Override + public List listSnapshotOnCache(long snapshotId) { + List cacheSnapshots = snapshotStoreDao.listOnCache(snapshotId); + List snapObjs = new ArrayList(); + for (SnapshotDataStoreVO cacheSnap : cacheSnapshots) { + long storeId = cacheSnap.getDataStoreId(); + DataStore store = storeMgr.getDataStore(storeId, DataStoreRole.ImageCache); + SnapshotInfo tmplObj = getSnapshot(snapshotId, store); + snapObjs.add(tmplObj); + } + return snapObjs; + } + } diff --git a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java index 15335bfb8cd..7eec5ffbb23 100644 --- a/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java +++ b/engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java @@ -16,17 +16,14 @@ // under the License. package org.apache.cloudstack.storage.snapshot; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Snapshot; -import com.cloud.storage.SnapshotVO; -import com.cloud.storage.Volume; -import com.cloud.storage.dao.SnapshotDao; -import com.cloud.storage.snapshot.SnapshotManager; -import com.cloud.utils.NumbersUtil; -import com.cloud.utils.db.DB; -import com.cloud.utils.exception.CloudRuntimeException; -import com.cloud.utils.fsm.NoTransitionException; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; @@ -42,10 +39,18 @@ import org.apache.cloudstack.storage.command.CreateObjectAnswer; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; -import javax.inject.Inject; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Volume; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.snapshot.SnapshotManager; +import com.cloud.utils.NumbersUtil; +import com.cloud.utils.db.DB; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.fsm.NoTransitionException; @Component public class XenserverSnapshotStrategy extends SnapshotStrategyBase { @@ -71,7 +76,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { if (parentSnapshot != null && snapshot.getPath().equalsIgnoreCase(parentSnapshot.getPath())) { s_logger.debug("backup an empty snapshot"); // don't need to backup this snapshot - SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot( + SnapshotDataStoreVO parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot( parentSnapshot.getId(), DataStoreRole.Image); if (parentSnapshotOnBackupStore != null && parentSnapshotOnBackupStore.getState() == State.Ready) { DataStore store = dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(), @@ -93,7 +98,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { s_logger.debug("Failed to change state: " + snapshot.getId() + ": " + e.toString()); throw new CloudRuntimeException(e.toString()); } - return this.snapshotDataFactory.getSnapshot(snapObj.getId(), store); + return snapshotDataFactory.getSnapshot(snapObj.getId(), store); } else { s_logger.debug("parent snapshot hasn't been backed up yet"); } @@ -111,7 +116,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { int i; SnapshotDataStoreVO parentSnapshotOnBackupStore = null; for (i = 1; i < deltaSnap; i++) { - parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), + parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(parentSnapshot.getId(), DataStoreRole.Image); if (parentSnapshotOnBackupStore == null) { break; @@ -122,7 +127,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { break; } - parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); + parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image); } if (i >= deltaSnap) { fullBackup = true; @@ -130,7 +135,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } snapshot.addPayload(fullBackup); - return this.snapshotSvr.backupSnapshot(snapshot); + return snapshotSvr.backupSnapshot(snapshot); } protected boolean deleteSnapshotChain(SnapshotInfo snapshot) { @@ -164,7 +169,15 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { } } if (!deleted) { - boolean r = this.snapshotSvr.deleteSnapshot(snapshot); + boolean r = snapshotSvr.deleteSnapshot(snapshot); + if (r) { + // delete snapshot in cache if there is + List cacheSnaps = snapshotDataFactory.listSnapshotOnCache(snapshot.getId()); + for (SnapshotInfo cacheSnap : cacheSnaps) { + s_logger.debug("Delete snapshot " + snapshot.getId() + " from image cache store: " + cacheSnap.getDataStore().getName()); + cacheSnap.delete(); + } + } if (!resultIsSet) { result = r; resultIsSet = true; @@ -200,7 +213,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { // first mark the snapshot as destroyed, so that ui can't see it, but we // may not destroy the snapshot on the storage, as other snapshots may // depend on it. - SnapshotInfo snapshotOnImage = this.snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image); + SnapshotInfo snapshotOnImage = snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image); if (snapshotOnImage == null) { s_logger.debug("Can't find snapshot on backup storage, delete it in db"); snapshotDao.remove(snapshotId); @@ -273,7 +286,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase { snapshot = result.getSnashot(); DataStore primaryStore = snapshot.getDataStore(); - SnapshotInfo backupedSnapshot = this.backupSnapshot(snapshot); + SnapshotInfo backupedSnapshot = backupSnapshot(snapshot); try { SnapshotInfo parent = snapshot.getParent(); diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java index a9263a98879..88061aa5e12 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java @@ -107,6 +107,11 @@ public class DataStoreManagerImpl implements DataStoreManager { return imageDataStoreMgr.listImageStores(); } + @Override + public List listImageCacheStores() { + return imageDataStoreMgr.listImageCacheStores(); + } + public void setPrimaryStoreMgr(PrimaryDataStoreProviderManager primaryStoreMgr) { this.primaryStoreMgr = primaryStoreMgr; } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java index a64114691c8..e2c48ea51e8 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java @@ -24,11 +24,14 @@ import java.util.UUID; import javax.inject.Inject; +import org.springframework.stereotype.Component; + +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.springframework.stereotype.Component; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; @@ -40,6 +43,8 @@ public class ImageStoreHelper { ImageStoreDao imageStoreDao; @Inject ImageStoreDetailsDao imageStoreDetailsDao; + @Inject + SnapshotDataStoreDao snapshotStoreDao; public ImageStoreVO createImageStore(Map params) { ImageStoreVO store = imageStoreDao.findByName((String) params.get("name")); @@ -115,4 +120,18 @@ public class ImageStoreHelper { imageStoreDao.remove(id); return true; } + + /** + * Convert current NFS secondary storage to Staging store to be ready to migrate to S3 object store. + * @param store NFS image store. + * @return true if successful. + */ + public boolean convertToStagingStore(DataStore store) { + ImageStoreVO nfsStore = imageStoreDao.findById(store.getId()); + nfsStore.setRole(DataStoreRole.ImageCache); + imageStoreDao.update(store.getId(), nfsStore); + // clear snapshot entry on primary store to make next snapshot become full snapshot + snapshotStoreDao.deleteSnapshotRecordsOnPrimary(); + return true; + } } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java index be66cc51401..8afb3d9f31a 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java @@ -32,6 +32,8 @@ public interface ImageStoreProviderManager { List listImageStores(); + List listImageCacheStores(); + List listImageStoresByScope(ZoneScope scope); List listImageStoreByProvider(String provider); diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java index b9ef9c307af..13a7f470b0f 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java @@ -23,10 +23,11 @@ import java.util.Map; import javax.naming.ConfigurationException; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; -import org.springframework.stereotype.Component; import com.cloud.storage.DataStoreRole; import com.cloud.storage.ScopeType; @@ -118,4 +119,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase implem return listBy(sc); } + @Override + public List listImageCacheStores() { + SearchCriteria sc = createSearchCriteria(); + sc.addAnd("role", SearchCriteria.Op.EQ, DataStoreRole.ImageCache); + return listBy(sc); + } + } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java index 48416a2d96c..ee00dd5d0e9 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java @@ -25,23 +25,23 @@ import java.util.Map; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; -import com.cloud.utils.db.DB; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.storage.DataStoreRole; +import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.UpdateBuilder; @Component @@ -54,7 +54,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase snapshotSearch; private SearchBuilder storeSnapshotSearch; private SearchBuilder snapshotIdSearch; - private String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " + + private final String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " + " and store_role = ? and volume_id = ? and state = 'Ready'" + " order by created DESC " + " limit 1"; @@ -170,6 +170,16 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase sc = storeSearch.create(); + sc.setParameters("store_role", DataStoreRole.Primary); + TransactionLegacy txn = TransactionLegacy.currentTxn(); + txn.start(); + remove(sc); + txn.commit(); + } + @Override public SnapshotDataStoreVO findByStoreSnapshot(DataStoreRole role, long storeId, long snapshotId) { SearchCriteria sc = storeSnapshotSearch.create(); @@ -195,7 +205,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase sc = storeSnapshotSearch.create(); + sc.setParameters("store_role", DataStoreRole.ImageCache); + sc.setParameters("destroyed", false); + List snapshots = listBy(sc); + // create an entry for each record, but with empty install path since the content is not yet on region-wide store yet + if (snapshots != null) { + s_logger.info("Duplicate " + snapshots.size() + " snapshot cache store records to region store"); + for (SnapshotDataStoreVO snap : snapshots) { + SnapshotDataStoreVO snapStore = findByStoreSnapshot(DataStoreRole.Image, storeId, snap.getSnapshotId()); + if (snapStore != null) { + s_logger.info("There is already entry for snapshot " + snap.getSnapshotId() + " on region store " + storeId); + continue; + } + s_logger.info("Persisting an entry for snapshot " + snap.getSnapshotId() + " on region store " + storeId); + SnapshotDataStoreVO ss = new SnapshotDataStoreVO(); + ss.setSnapshotId(snap.getSnapshotId()); + ss.setDataStoreId(storeId); + ss.setRole(DataStoreRole.Image); + ss.setVolumeId(snap.getVolumeId()); + ss.setParentSnapshotId(snap.getParentSnapshotId()); + ss.setState(snap.getState()); + ss.setSize(snap.getSize()); + ss.setPhysicalSize(snap.getPhysicalSize()); + ss.setRefCnt(snap.getRefCnt()); + persist(ss); + // increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store + snap.incrRefCnt(); + update(snap.getId(), snap); + } + } + + } + + @Override + public List listOnCache(long snapshotId) { + SearchCriteria sc = storeSnapshotSearch.create(); + sc.setParameters("snapshot_id", snapshotId); + sc.setParameters("store_role", DataStoreRole.ImageCache); + return search(sc, null); + } + + @Override + public void updateStoreRoleToCache(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("destroyed", false); + List snaps = listBy(sc); + if (snaps != null) { + s_logger.info("Update to cache store role for " + snaps.size() + " entries in snapshot_store_ref"); + for (SnapshotDataStoreVO snap : snaps) { + snap.setRole(DataStoreRole.ImageCache); + update(snap.getId(), snap); + } + } + } + } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java index 4509e233e81..ee7c4fcfd56 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java @@ -31,20 +31,26 @@ import org.springframework.stereotype.Component; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; +import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService; import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.dao.VMTemplateDao; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.UpdateBuilder; +import com.cloud.utils.exception.CloudRuntimeException; @Component public class TemplateDataStoreDaoImpl extends GenericDaoBase implements TemplateDataStoreDao { @@ -61,6 +67,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase params) throws ConfigurationException { super.configure(name, params); @@ -85,6 +96,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase sc = templateRoleSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("store_role", DataStoreRole.ImageCache); + sc.setParameters("destroyed", false); + sc.setParameters("state", ObjectInDataStoreStateMachine.State.Ready); + return findOneIncludingRemovedBy(sc); + } + + @Override + public List listOnCache(long templateId) { + SearchCriteria sc = templateRoleSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("store_role", DataStoreRole.ImageCache); + return search(sc, null); + } + @Override public List listByTemplate(long templateId) { SearchCriteria sc = templateSearch.create(); @@ -341,4 +371,78 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase sc = templateRoleSearch.create(); + sc.setParameters("store_role", DataStoreRole.ImageCache); + sc.setParameters("destroyed", false); + List tmpls = listBy(sc); + // create an entry for each template record, but with empty install path since the content is not yet on region-wide store yet + if (tmpls != null) { + s_logger.info("Duplicate " + tmpls.size() + " template cache store records to region store"); + for (TemplateDataStoreVO tmpl : tmpls) { + long templateId = tmpl.getTemplateId(); + VMTemplateVO template = _tmpltDao.findById(templateId); + if (template == null) { + throw new CloudRuntimeException("No template is found for template id: " + templateId); + } + if (template.getTemplateType() == TemplateType.SYSTEM) { + s_logger.info("No need to duplicate system template since it will be automatically downloaded while adding region store"); + continue; + } + TemplateDataStoreVO tmpStore = findByStoreTemplate(storeId, tmpl.getTemplateId()); + if (tmpStore != null) { + s_logger.info("There is already entry for template " + tmpl.getTemplateId() + " on region store " + storeId); + continue; + } + s_logger.info("Persisting an entry for template " + tmpl.getTemplateId() + " on region store " + storeId); + TemplateDataStoreVO ts = new TemplateDataStoreVO(); + ts.setTemplateId(tmpl.getTemplateId()); + ts.setDataStoreId(storeId); + ts.setDataStoreRole(DataStoreRole.Image); + ts.setState(tmpl.getState()); + ts.setDownloadPercent(tmpl.getDownloadPercent()); + ts.setDownloadState(tmpl.getDownloadState()); + ts.setSize(tmpl.getSize()); + ts.setPhysicalSize(tmpl.getPhysicalSize()); + ts.setErrorString(tmpl.getErrorString()); + ts.setDownloadUrl(tmpl.getDownloadUrl()); + ts.setRefCnt(tmpl.getRefCnt()); + persist(ts); + // increase ref_cnt of cache store entry so that this will not be recycled before the content is pushed to region-wide store + tmpl.incrRefCnt(); + this.update(tmpl.getId(), tmpl); + + // mark the template as cross-zones + template.setCrossZones(true); + _tmpltDao.update(templateId, template); + // add template_zone_ref association for these cross-zone templates + _tmplSrv.associateTemplateToZone(templateId, null); + } + + } + } + + + @Override + public void updateStoreRoleToCachce(long storeId) { + SearchCriteria sc = storeSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("destroyed", false); + List tmpls = listBy(sc); + if (tmpls != null) { + s_logger.info("Update to cache store role for " + tmpls.size() + " entries in template_store_ref"); + for (TemplateDataStoreVO tmpl : tmpls) { + tmpl.setDataStoreRole(DataStoreRole.ImageCache); + update(tmpl.getId(), tmpl); + } + } + + } + + } diff --git a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java index 2bacd03560a..a3ff5666a7d 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java @@ -16,25 +16,30 @@ // under the License. package org.apache.cloudstack.storage.image.db; +import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; +import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; -import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.SearchCriteria.Op; +import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.db.UpdateBuilder; @Component @@ -45,6 +50,9 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase storeSearch; private SearchBuilder cacheSearch; private SearchBuilder storeVolumeSearch; + + @Inject + DataStoreManager storeMgr; @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -186,4 +194,45 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase cacheStores = storeMgr.listImageCacheStores(); + if (cacheStores == null || cacheStores.size() == 0) { + return; + } + List vols = new ArrayList(); + for (DataStore store : cacheStores) { + // check if the volume is stored there + vols.addAll(listByStoreId(store.getId())); + } + // create an entry for each record, but with empty install path since the content is not yet on region-wide store yet + if (vols != null) { + s_logger.info("Duplicate " + vols.size() + " volume cache store records to region store"); + for (VolumeDataStoreVO vol : vols) { + VolumeDataStoreVO volStore = findByStoreVolume(storeId, vol.getVolumeId()); + if (volStore != null) { + s_logger.info("There is already entry for volume " + vol.getVolumeId() + " on region store " + storeId); + continue; + } + s_logger.info("Persisting an entry for volume " + vol.getVolumeId() + " on region store " + storeId); + VolumeDataStoreVO vs = new VolumeDataStoreVO(); + vs.setVolumeId(vol.getVolumeId()); + vs.setDataStoreId(storeId); + vs.setState(vol.getState()); + vs.setDownloadPercent(vol.getDownloadPercent()); + vs.setDownloadState(vol.getDownloadState()); + vs.setSize(vol.getSize()); + vs.setPhysicalSize(vol.getPhysicalSize()); + vs.setErrorString(vol.getErrorString()); + vs.setRefCnt(vol.getRefCnt()); + persist(vs); + // increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store + vol.incrRefCnt(); + this.update(vol.getId(), vol); + } + } + + } } diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java index 1d75ba1529b..8065f9c9378 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java @@ -18,8 +18,13 @@ */ package org.apache.cloudstack.storage.volume; +import java.util.ArrayList; +import java.util.List; + import javax.inject.Inject; +import org.springframework.stereotype.Component; + import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -27,7 +32,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; -import org.springframework.stereotype.Component; import com.cloud.storage.DataStoreRole; import com.cloud.storage.VolumeVO; @@ -58,13 +62,13 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { if (storeRole == DataStoreRole.Image) { VolumeDataStoreVO volumeStore = volumeStoreDao.findByVolume(volumeId); if (volumeStore != null) { - DataStore store = this.storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image); + DataStore store = storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image); vol = VolumeObject.getVolumeObject(store, volumeVO); } } else { // Primary data store if (volumeVO.getPoolId() != null) { - DataStore store = this.storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary); + DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary); vol = VolumeObject.getVolumeObject(store, volumeVO); } } @@ -82,11 +86,11 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { DataStore store = null; VolumeDataStoreVO volumeStore = volumeStoreDao.findByVolume(volumeId); if (volumeStore != null) { - store = this.storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image); + store = storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image); } vol = VolumeObject.getVolumeObject(store, volumeVO); } else { - DataStore store = this.storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary); + DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary); vol = VolumeObject.getVolumeObject(store, volumeVO); } return vol; @@ -99,4 +103,23 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory { return vol; } + @Override + public List listVolumeOnCache(long volumeId) { + List cacheVols = new ArrayList(); + // find all image cache stores for this zone scope + List cacheStores = storeMgr.listImageCacheStores(); + if (cacheStores == null || cacheStores.size() == 0) { + return cacheVols; + } + for (DataStore store : cacheStores) { + // check if the volume is stored there + VolumeDataStoreVO volStore = volumeStoreDao.findByStoreVolume(store.getId(), volumeId); + if (volStore != null) { + VolumeInfo vol = getVolume(volumeId, store); + cacheVols.add(vol); + } + } + return cacheVols; + } + } diff --git a/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorImageStoreLifeCycleImpl.java b/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorImageStoreLifeCycleImpl.java index beaa7a5fbae..09644141a53 100644 --- a/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorImageStoreLifeCycleImpl.java +++ b/plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorImageStoreLifeCycleImpl.java @@ -20,12 +20,15 @@ package org.apache.cloudstack.storage.datastore.lifecycle; -import com.cloud.agent.api.StoragePoolInfo; -import com.cloud.exception.InvalidParameterValueException; -import com.cloud.hypervisor.Hypervisor; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.ScopeType; -import com.cloud.utils.UriUtils; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; @@ -34,13 +37,13 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle; -import org.apache.log4j.Logger; -import javax.inject.Inject; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.HashMap; -import java.util.Map; +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; +import com.cloud.utils.UriUtils; public class SimulatorImageStoreLifeCycleImpl implements ImageStoreLifeCycle { private static final Logger s_logger = Logger.getLogger(SimulatorImageStoreLifeCycleImpl.class); @@ -129,4 +132,13 @@ public class SimulatorImageStoreLifeCycleImpl implements ImageStoreLifeCycle { public boolean deleteDataStore(DataStore store) { return false; } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java index ee6f47b8fd1..d6448785a93 100644 --- a/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java +++ b/plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java @@ -161,4 +161,13 @@ public class CloudStackImageStoreLifeCycleImpl implements ImageStoreLifeCycle { public boolean deleteDataStore(DataStore store) { return false; } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return imageStoreHelper.convertToStagingStore(store); + } + } diff --git a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java index 249a4c6a1e8..de25830822f 100644 --- a/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java +++ b/plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java @@ -22,6 +22,8 @@ import java.util.Map; import javax.inject.Inject; +import org.apache.log4j.Logger; + import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -32,7 +34,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle; -import org.apache.log4j.Logger; import com.cloud.agent.api.StoragePoolInfo; import com.cloud.hypervisor.Hypervisor.HypervisorType; @@ -131,4 +132,13 @@ public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle { public boolean deleteDataStore(DataStore store) { return false; } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/plugins/storage/image/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SampleImageStoreLifeCycleImpl.java b/plugins/storage/image/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SampleImageStoreLifeCycleImpl.java index e4df6f55f3a..b10b756d11f 100644 --- a/plugins/storage/image/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SampleImageStoreLifeCycleImpl.java +++ b/plugins/storage/image/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SampleImageStoreLifeCycleImpl.java @@ -79,4 +79,13 @@ public class SampleImageStoreLifeCycleImpl implements ImageStoreLifeCycle { public boolean deleteDataStore(DataStore store) { return false; } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/lifecycle/SwiftImageStoreLifeCycleImpl.java b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/lifecycle/SwiftImageStoreLifeCycleImpl.java index 4256cc2cc7b..0a2b72c3aa3 100644 --- a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/lifecycle/SwiftImageStoreLifeCycleImpl.java +++ b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/lifecycle/SwiftImageStoreLifeCycleImpl.java @@ -16,11 +16,13 @@ // under the License. package org.apache.cloudstack.storage.datastore.lifecycle; -import com.cloud.agent.api.StoragePoolInfo; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.resource.ResourceManager; -import com.cloud.storage.DataStoreRole; -import com.cloud.storage.ScopeType; +import java.util.HashMap; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; @@ -30,11 +32,12 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper; import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager; import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle; -import org.apache.log4j.Logger; -import javax.inject.Inject; -import java.util.HashMap; -import java.util.Map; +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.resource.ResourceManager; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.ScopeType; public class SwiftImageStoreLifeCycleImpl implements ImageStoreLifeCycle { @@ -113,4 +116,13 @@ public class SwiftImageStoreLifeCycleImpl implements ImageStoreLifeCycle { public boolean deleteDataStore(DataStore store) { return false; } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java index 5e0dd958f34..d916d454422 100644 --- a/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java @@ -433,7 +433,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore List poolHosts = new ArrayList(); for (HostVO host : hosts) { try { - this.storageMgr.connectHostToSharedPool(host.getId(), dataStore.getId()); + storageMgr.connectHostToSharedPool(host.getId(), dataStore.getId()); poolHosts.add(host); } catch (Exception e) { s_logger.warn("Unable to establish a connection between " + host + " and " + dataStore, e); @@ -444,20 +444,20 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore primaryDataStoreDao.expunge(dataStore.getId()); throw new CloudRuntimeException("Failed to create storage pool as it is not accessible to hosts."); } - this.dataStoreHelper.attachZone(dataStore, hypervisorType); + dataStoreHelper.attachZone(dataStore, hypervisorType); return true; } @Override public boolean maintain(DataStore dataStore) { storagePoolAutmation.maintain(dataStore); - this.dataStoreHelper.maintain(dataStore); + dataStoreHelper.maintain(dataStore); return true; } @Override public boolean cancelMaintain(DataStore store) { - this.dataStoreHelper.cancelMaintain(store); + dataStoreHelper.cancelMaintain(store); storagePoolAutmation.cancelMaintain(store); return true; } @@ -513,4 +513,13 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore dataStoreHelper.attachHost(store, scope, existingInfo); return true; } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SamplePrimaryDataStoreLifeCycleImpl.java b/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SamplePrimaryDataStoreLifeCycleImpl.java index 92538ad5f4b..6b5e43127b8 100644 --- a/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SamplePrimaryDataStoreLifeCycleImpl.java +++ b/plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SamplePrimaryDataStoreLifeCycleImpl.java @@ -18,12 +18,18 @@ */ package org.apache.cloudstack.storage.datastore.lifecycle; -import com.cloud.agent.api.StoragePoolInfo; -import com.cloud.host.HostVO; -import com.cloud.host.dao.HostDao; -import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.storage.StoragePoolStatus; -import org.apache.cloudstack.engine.subsystem.api.storage.*; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; + +import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; +import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.command.AttachPrimaryDataStoreCmd; import org.apache.cloudstack.storage.command.CreatePrimaryDataStoreCmd; import org.apache.cloudstack.storage.datastore.PrimaryDataStoreProviderManager; @@ -31,9 +37,11 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; -import javax.inject.Inject; -import java.util.List; -import java.util.Map; +import com.cloud.agent.api.StoragePoolInfo; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.storage.StoragePoolStatus; public class SamplePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle { @Inject @@ -119,4 +127,12 @@ public class SamplePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLife return false; } + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java b/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java index 038e89b5794..120a357b270 100644 --- a/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java +++ b/plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java @@ -25,27 +25,28 @@ import java.util.StringTokenizer; import javax.inject.Inject; +import org.apache.log4j.Logger; + import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HostScope; -import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle; import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters; +import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.util.SolidFireUtil; import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper; -import org.apache.log4j.Logger; +import com.cloud.agent.api.StoragePoolInfo; import com.cloud.dc.DataCenterVO; import com.cloud.dc.dao.DataCenterDao; import com.cloud.host.HostVO; import com.cloud.hypervisor.Hypervisor.HypervisorType; -import com.cloud.agent.api.StoragePoolInfo; import com.cloud.resource.ResourceManager; +import com.cloud.storage.Storage.StoragePoolType; import com.cloud.storage.StorageManager; import com.cloud.storage.StoragePoolAutomation; -import com.cloud.storage.Storage.StoragePoolType; import com.cloud.utils.exception.CloudRuntimeException; public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCycle { @@ -58,39 +59,39 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC @Inject StorageManager _storageMgr; @Inject private StoragePoolAutomation storagePoolAutomation; @Inject private StoragePoolDetailsDao storagePoolDetailsDao; - + private static final int DEFAULT_MANAGEMENT_PORT = 443; private static final int DEFAULT_STORAGE_PORT = 3260; - + // invoked to add primary storage that is based on the SolidFire plug-in @Override public DataStore initialize(Map dsInfos) { - String url = (String)dsInfos.get("url"); - Long zoneId = (Long)dsInfos.get("zoneId"); + String url = (String)dsInfos.get("url"); + Long zoneId = (Long)dsInfos.get("zoneId"); String storagePoolName = (String) dsInfos.get("name"); String providerName = (String)dsInfos.get("providerName"); Long capacityBytes = (Long)dsInfos.get("capacityBytes"); Long capacityIops = (Long)dsInfos.get("capacityIops"); String tags = (String)dsInfos.get("tags"); Map details = (Map)dsInfos.get("details"); - - String storageVip = getStorageVip(url); - int storagePort = getStoragePort(url); - - DataCenterVO zone = zoneDao.findById(zoneId); - - String uuid = SolidFireUtil.PROVIDER_NAME + "_" + zone.getUuid() + "_" + storageVip; + + String storageVip = getStorageVip(url); + int storagePort = getStoragePort(url); + + DataCenterVO zone = zoneDao.findById(zoneId); + + String uuid = SolidFireUtil.PROVIDER_NAME + "_" + zone.getUuid() + "_" + storageVip; if (capacityBytes == null || capacityBytes <= 0) { throw new IllegalArgumentException("'capacityBytes' must be present and greater than 0."); } - - if (capacityIops == null || capacityIops <= 0) { - throw new IllegalArgumentException("'capacityIops' must be present and greater than 0."); - } - + + if (capacityIops == null || capacityIops <= 0) { + throw new IllegalArgumentException("'capacityIops' must be present and greater than 0."); + } + PrimaryDataStoreParameters parameters = new PrimaryDataStoreParameters(); - + parameters.setHost(storageVip); parameters.setPort(storagePort); parameters.setPath(getModifiedUrl(url)); @@ -106,16 +107,16 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC parameters.setHypervisorType(HypervisorType.Any); parameters.setTags(tags); parameters.setDetails(details); - + String managementVip = getManagementVip(url); int managementPort = getManagementPort(url); - + details.put(SolidFireUtil.MANAGEMENT_VIP, managementVip); details.put(SolidFireUtil.MANAGEMENT_PORT, String.valueOf(managementPort)); - + String clusterAdminUsername = getValue(SolidFireUtil.CLUSTER_ADMIN_USERNAME, url); String clusterAdminPassword = getValue(SolidFireUtil.CLUSTER_ADMIN_PASSWORD, url); - + details.put(SolidFireUtil.CLUSTER_ADMIN_USERNAME, clusterAdminUsername); details.put(SolidFireUtil.CLUSTER_ADMIN_PASSWORD, clusterAdminPassword); @@ -181,140 +182,140 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC // this adds a row in the cloud.storage_pool table for this SolidFire cluster return dataStoreHelper.createPrimaryDataStore(parameters); } - + // remove the clusterAdmin and password key/value pairs private String getModifiedUrl(String originalUrl) { - StringBuilder sb = new StringBuilder(); - - String delimiter = ";"; - - StringTokenizer st = new StringTokenizer(originalUrl, delimiter); - - while (st.hasMoreElements()) { - String token = st.nextElement().toString().toUpperCase(); - - if (token.startsWith(SolidFireUtil.MANAGEMENT_VIP.toUpperCase()) || - token.startsWith(SolidFireUtil.STORAGE_VIP.toUpperCase())) { - sb.append(token).append(delimiter); - } - } - - String modifiedUrl = sb.toString(); - int lastIndexOf = modifiedUrl.lastIndexOf(delimiter); - - if (lastIndexOf == (modifiedUrl.length() - delimiter.length())) { - return modifiedUrl.substring(0, lastIndexOf); - } - - return modifiedUrl; + StringBuilder sb = new StringBuilder(); + + String delimiter = ";"; + + StringTokenizer st = new StringTokenizer(originalUrl, delimiter); + + while (st.hasMoreElements()) { + String token = st.nextElement().toString().toUpperCase(); + + if (token.startsWith(SolidFireUtil.MANAGEMENT_VIP.toUpperCase()) || + token.startsWith(SolidFireUtil.STORAGE_VIP.toUpperCase())) { + sb.append(token).append(delimiter); + } + } + + String modifiedUrl = sb.toString(); + int lastIndexOf = modifiedUrl.lastIndexOf(delimiter); + + if (lastIndexOf == (modifiedUrl.length() - delimiter.length())) { + return modifiedUrl.substring(0, lastIndexOf); + } + + return modifiedUrl; } - + private String getManagementVip(String url) { return getVip(SolidFireUtil.MANAGEMENT_VIP, url); } - + private String getStorageVip(String url) { return getVip(SolidFireUtil.STORAGE_VIP, url); } - + private int getManagementPort(String url) { return getPort(SolidFireUtil.MANAGEMENT_VIP, url, DEFAULT_MANAGEMENT_PORT); } - + private int getStoragePort(String url) { return getPort(SolidFireUtil.STORAGE_VIP, url, DEFAULT_STORAGE_PORT); } - + private String getVip(String keyToMatch, String url) { - String delimiter = ":"; - - String storageVip = getValue(keyToMatch, url); - - int index = storageVip.indexOf(delimiter); - - if (index != -1) - { - return storageVip.substring(0, index); - } - - return storageVip; + String delimiter = ":"; + + String storageVip = getValue(keyToMatch, url); + + int index = storageVip.indexOf(delimiter); + + if (index != -1) + { + return storageVip.substring(0, index); + } + + return storageVip; } - + private int getPort(String keyToMatch, String url, int defaultPortNumber) { - String delimiter = ":"; - - String storageVip = getValue(keyToMatch, url); - - int index = storageVip.indexOf(delimiter); - - int portNumber = defaultPortNumber; - - if (index != -1) { - String port = storageVip.substring(index + delimiter.length()); - - try { - portNumber = Integer.parseInt(port); - } - catch (NumberFormatException ex) { - throw new IllegalArgumentException("Invalid URL format (port is not an integer)"); - } - } - - return portNumber; + String delimiter = ":"; + + String storageVip = getValue(keyToMatch, url); + + int index = storageVip.indexOf(delimiter); + + int portNumber = defaultPortNumber; + + if (index != -1) { + String port = storageVip.substring(index + delimiter.length()); + + try { + portNumber = Integer.parseInt(port); + } + catch (NumberFormatException ex) { + throw new IllegalArgumentException("Invalid URL format (port is not an integer)"); + } + } + + return portNumber; } - + private String getValue(String keyToMatch, String url) { - String delimiter1 = ";"; - String delimiter2 = "="; - - StringTokenizer st = new StringTokenizer(url, delimiter1); - - while (st.hasMoreElements()) { - String token = st.nextElement().toString(); - - int index = token.indexOf(delimiter2); - - if (index == -1) - { - throw new RuntimeException("Invalid URL format"); - } - - String key = token.substring(0, index); - - if (key.equalsIgnoreCase(keyToMatch)) { - String valueToReturn = token.substring(index + delimiter2.length()); - - return valueToReturn; - } - } - - throw new RuntimeException("Key not found in URL"); + String delimiter1 = ";"; + String delimiter2 = "="; + + StringTokenizer st = new StringTokenizer(url, delimiter1); + + while (st.hasMoreElements()) { + String token = st.nextElement().toString(); + + int index = token.indexOf(delimiter2); + + if (index == -1) + { + throw new RuntimeException("Invalid URL format"); + } + + String key = token.substring(0, index); + + if (key.equalsIgnoreCase(keyToMatch)) { + String valueToReturn = token.substring(index + delimiter2.length()); + + return valueToReturn; + } + } + + throw new RuntimeException("Key not found in URL"); } - + // do not implement this method for SolidFire's plug-in @Override public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) { return true; // should be ignored for zone-wide-only plug-ins like SolidFire's } - + // do not implement this method for SolidFire's plug-in @Override public boolean attachCluster(DataStore store, ClusterScope scope) { return true; // should be ignored for zone-wide-only plug-ins like SolidFire's } - + @Override public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) { - dataStoreHelper.attachZone(dataStore); - + dataStoreHelper.attachZone(dataStore); + List xenServerHosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType.XenServer, scope.getScopeId()); List kvmHosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType.KVM, scope.getScopeId()); List hosts = new ArrayList(); @@ -328,11 +329,11 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC } catch (Exception e) { s_logger.warn("Unable to establish a connection between " + host + " and " + dataStore, e); } - } + } return true; } - + @Override public boolean maintain(DataStore dataStore) { storagePoolAutomation.maintain(dataStore); @@ -340,7 +341,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC return true; } - + @Override public boolean cancelMaintain(DataStore store) { dataStoreHelper.cancelMaintain(store); @@ -348,10 +349,19 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC return true; } - + // invoked to delete primary storage that is based on the SolidFire plug-in @Override public boolean deleteDataStore(DataStore store) { return dataStoreHelper.deletePrimaryDataStore(store); } + + /* (non-Javadoc) + * @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore) + */ + @Override + public boolean migrateToObjectStore(DataStore store) { + return false; + } + } diff --git a/server/src/com/cloud/server/ManagementServerImpl.java b/server/src/com/cloud/server/ManagementServerImpl.java index 058a7568da0..79b20d008b6 100755 --- a/server/src/com/cloud/server/ManagementServerImpl.java +++ b/server/src/com/cloud/server/ManagementServerImpl.java @@ -42,6 +42,9 @@ import javax.crypto.spec.SecretKeySpec; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.ControlledEntity; import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.affinity.AffinityGroupProcessor; @@ -156,6 +159,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStore import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd; import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForMaintenanceCmd; +import org.apache.cloudstack.api.command.admin.storage.PrepareSecondaryStorageForMigrationCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; @@ -437,8 +441,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.identity.ManagementServerNode; -import org.apache.commons.codec.binary.Base64; -import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.GetVncPortAnswer; @@ -2225,21 +2227,21 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - if (domainName != null) { - String updatedDomainPath = getUpdatedDomainPath(domain.getPath(), domainName); - updateDomainChildren(domain, updatedDomainPath); - domain.setName(domainName); - domain.setPath(updatedDomainPath); - } - - if (networkDomain != null) { - if (networkDomain.isEmpty()) { - domain.setNetworkDomain(null); - } else { - domain.setNetworkDomain(networkDomain); - } - } - _domainDao.update(domainId, domain); + if (domainName != null) { + String updatedDomainPath = getUpdatedDomainPath(domain.getPath(), domainName); + updateDomainChildren(domain, updatedDomainPath); + domain.setName(domainName); + domain.setPath(updatedDomainPath); + } + + if (networkDomain != null) { + if (networkDomain.isEmpty()) { + domain.setNetworkDomain(null); + } else { + domain.setNetworkDomain(networkDomain); + } + } + _domainDao.update(domainId, domain); } }); @@ -2840,6 +2842,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(CreateSecondaryStagingStoreCmd.class); cmdList.add(ListSecondaryStagingStoresCmd.class); cmdList.add(DeleteSecondaryStagingStoreCmd.class); + cmdList.add(PrepareSecondaryStorageForMigrationCmd.class); cmdList.add(CreateApplicationLoadBalancerCmd.class); cmdList.add(ListApplicationLoadBalancersCmd.class); cmdList.add(DeleteApplicationLoadBalancerCmd.class); @@ -3683,24 +3686,24 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - for (HostVO h : hosts) { - if (s_logger.isDebugEnabled()) { - s_logger.debug("Changing password for host name = " + h.getName()); - } - // update password for this host - DetailVO nv = _detailsDao.findDetail(h.getId(), ApiConstants.USERNAME); - if (nv.getValue().equals(cmd.getUsername())) { - DetailVO nvp = _detailsDao.findDetail(h.getId(), ApiConstants.PASSWORD); - nvp.setValue(DBEncryptionUtil.encrypt(cmd.getPassword())); - _detailsDao.persist(nvp); - } else { - // if one host in the cluster has diff username then - // rollback to maintain consistency - throw new InvalidParameterValueException( - "The username is not same for all hosts, please modify passwords for individual hosts."); - } + for (HostVO h : hosts) { + if (s_logger.isDebugEnabled()) { + s_logger.debug("Changing password for host name = " + h.getName()); + } + // update password for this host + DetailVO nv = _detailsDao.findDetail(h.getId(), ApiConstants.USERNAME); + if (nv.getValue().equals(cmd.getUsername())) { + DetailVO nvp = _detailsDao.findDetail(h.getId(), ApiConstants.PASSWORD); + nvp.setValue(DBEncryptionUtil.encrypt(cmd.getPassword())); + _detailsDao.persist(nvp); + } else { + // if one host in the cluster has diff username then + // rollback to maintain consistency + throw new InvalidParameterValueException( + "The username is not same for all hosts, please modify passwords for individual hosts."); } } + } }); } @@ -3883,7 +3886,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe @Inject public void setStoragePoolAllocators(List storagePoolAllocators) { - this._storagePoolAllocators = storagePoolAllocators; + _storagePoolAllocators = storagePoolAllocators; } public LockMasterListener getLockMasterListener() { diff --git a/server/src/com/cloud/storage/StorageManagerImpl.java b/server/src/com/cloud/storage/StorageManagerImpl.java index fe7863bd05d..73426531713 100755 --- a/server/src/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/com/cloud/storage/StorageManagerImpl.java @@ -43,6 +43,7 @@ import javax.naming.ConfigurationException; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; + import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd; @@ -158,15 +159,14 @@ import com.cloud.utils.db.DB; import com.cloud.utils.db.GenericSearchBuilder; import com.cloud.utils.db.GlobalLock; import com.cloud.utils.db.JoinBuilder; -import com.cloud.utils.db.TransactionCallback; -import com.cloud.utils.db.TransactionCallbackNoReturn; -import com.cloud.utils.db.TransactionLegacy; -import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.db.JoinBuilder.JoinType; import com.cloud.utils.db.SearchBuilder; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallbackNoReturn; +import com.cloud.utils.db.TransactionLegacy; +import com.cloud.utils.db.TransactionStatus; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine.State; @@ -1247,6 +1247,29 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return (PrimaryDataStoreInfo) dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); } + @Override + @DB + public ImageStore prepareSecondaryStorageForObjectStoreMigration(Long storeId) throws ResourceUnavailableException, InsufficientCapacityException { + // Verify that image store exists + ImageStoreVO store = _imageStoreDao.findById(storeId); + if (store == null) { + throw new InvalidParameterValueException("Image store with id " + storeId + " doesn't exist"); + } else if (!store.getProviderName().equals(DataStoreProvider.NFS_IMAGE)) { + throw new InvalidParameterValueException("We only support migrate NFS secondary storage to use object store!"); + } + _accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), store.getDataCenterId()); + + DataStoreProvider provider = dataStoreProviderMgr.getDataStoreProvider(store.getProviderName()); + DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle(); + DataStore secStore = dataStoreMgr.getDataStore(storeId, DataStoreRole.Image); + lifeCycle.migrateToObjectStore(secStore); + // update store_role in template_store_ref and snapshot_store_ref to ImageCache + _templateStoreDao.updateStoreRoleToCachce(storeId); + _snapshotStoreDao.updateStoreRoleToCache(storeId); + // converted to an image cache store + return (ImageStore)_dataStoreMgr.getDataStore(storeId, DataStoreRole.ImageCache); + } + protected class StorageGarbageCollector extends ManagedContextRunnable { public StorageGarbageCollector() { @@ -1722,9 +1745,20 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C // store associateCrosszoneTemplatesToZone(dcId); + // duplicate cache store records to region wide storage + if (scopeType == ScopeType.REGION) { + duplicateCacheStoreRecordsToRegionStore(store.getId()); + } + return (ImageStore) _dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Image); } + private void duplicateCacheStoreRecordsToRegionStore(long storeId) { + _templateStoreDao.duplicateCacheRecordsOnRegionStore(storeId); + _snapshotStoreDao.duplicateCacheRecordsOnRegionStore(storeId); + _volumeStoreDao.duplicateCacheRecordsOnRegionStore(storeId); + } + private void associateCrosszoneTemplatesToZone(Long zoneId) { VMTemplateZoneVO tmpltZone; @@ -1785,14 +1819,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - // first delete from image_store_details table, we need to do that since - // we are not actually deleting record from main - // image_data_store table, so delete cascade will not work - _imageStoreDetailsDao.deleteDetails(storeId); - _snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.Image); - _volumeStoreDao.deletePrimaryRecordsForStore(storeId); - _templateStoreDao.deletePrimaryRecordsForStore(storeId); - _imageStoreDao.remove(storeId); + // first delete from image_store_details table, we need to do that since + // we are not actually deleting record from main + // image_data_store table, so delete cascade will not work + _imageStoreDetailsDao.deleteDetails(storeId); + _snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.Image); + _volumeStoreDao.deletePrimaryRecordsForStore(storeId); + _templateStoreDao.deletePrimaryRecordsForStore(storeId); + _imageStoreDao.remove(storeId); } }); @@ -1897,14 +1931,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - // first delete from image_store_details table, we need to do that since - // we are not actually deleting record from main - // image_data_store table, so delete cascade will not work - _imageStoreDetailsDao.deleteDetails(storeId); - _snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.ImageCache); - _volumeStoreDao.deletePrimaryRecordsForStore(storeId); - _templateStoreDao.deletePrimaryRecordsForStore(storeId); - _imageStoreDao.remove(storeId); + // first delete from image_store_details table, we need to do that since + // we are not actually deleting record from main + // image_data_store table, so delete cascade will not work + _imageStoreDetailsDao.deleteDetails(storeId); + _snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.ImageCache); + _volumeStoreDao.deletePrimaryRecordsForStore(storeId); + _templateStoreDao.deletePrimaryRecordsForStore(storeId); + _imageStoreDao.remove(storeId); } }); diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index 67aa3888695..d445381f2dd 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -411,29 +411,29 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return Transaction.execute(new TransactionCallback() { @Override public VolumeVO doInTransaction(TransactionStatus status) { - VolumeVO volume = new VolumeVO(volumeName, zoneId, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK); - volume.setPoolId(null); - volume.setDataCenterId(zoneId); - volume.setPodId(null); - volume.setAccountId(owner.getAccountId()); - volume.setDomainId(owner.getDomainId()); - long diskOfferingId = _diskOfferingDao.findByUniqueName("Cloud.com-Custom").getId(); - volume.setDiskOfferingId(diskOfferingId); - // volume.setSize(size); - volume.setInstanceId(null); - volume.setUpdated(new Date()); - volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId()); - volume.setFormat(ImageFormat.valueOf(format)); - volume = _volsDao.persist(volume); - CallContext.current().setEventDetails("Volume Id: " + volume.getId()); - - // Increment resource count during allocation; if actual creation fails, - // decrement it - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); - - return volume; - } + VolumeVO volume = new VolumeVO(volumeName, zoneId, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK); + volume.setPoolId(null); + volume.setDataCenterId(zoneId); + volume.setPodId(null); + volume.setAccountId(owner.getAccountId()); + volume.setDomainId(owner.getDomainId()); + long diskOfferingId = _diskOfferingDao.findByUniqueName("Cloud.com-Custom").getId(); + volume.setDiskOfferingId(diskOfferingId); + // volume.setSize(size); + volume.setInstanceId(null); + volume.setUpdated(new Date()); + volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId()); + volume.setFormat(ImageFormat.valueOf(format)); + volume = _volsDao.persist(volume); + CallContext.current().setEventDetails("Volume Id: " + volume.getId()); + + // Increment resource count during allocation; if actual creation fails, + // decrement it + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + + return volume; + } }); } @@ -615,42 +615,42 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return Transaction.execute(new TransactionCallback() { @Override public VolumeVO doInTransaction(TransactionStatus status) { - VolumeVO volume = new VolumeVO(userSpecifiedName, -1, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK); - volume.setPoolId(null); - volume.setDataCenterId(zoneId); - volume.setPodId(null); - volume.setAccountId(ownerId); - volume.setDomainId(((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId())); - volume.setDiskOfferingId(diskOfferingId); - volume.setSize(size); - volume.setMinIops(minIops); - volume.setMaxIops(maxIops); - volume.setInstanceId(null); - volume.setUpdated(new Date()); - volume.setDomainId((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId()); - volume.setDisplayVolume(displayVolumeEnabled); - if (parentVolume != null) { - volume.setTemplateId(parentVolume.getTemplateId()); - volume.setFormat(parentVolume.getFormat()); - } else { - volume.setTemplateId(null); - } - - volume = _volsDao.persist(volume); - if (cmd.getSnapshotId() == null) { - // for volume created from snapshot, create usage event after volume creation - UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, - null, size, Volume.class.getName(), volume.getUuid()); - } - - CallContext.current().setEventDetails("Volume Id: " + volume.getId()); - - // Increment resource count during allocation; if actual creation fails, - // decrement it - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); - return volume; - } + VolumeVO volume = new VolumeVO(userSpecifiedName, -1, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK); + volume.setPoolId(null); + volume.setDataCenterId(zoneId); + volume.setPodId(null); + volume.setAccountId(ownerId); + volume.setDomainId(((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId())); + volume.setDiskOfferingId(diskOfferingId); + volume.setSize(size); + volume.setMinIops(minIops); + volume.setMaxIops(maxIops); + volume.setInstanceId(null); + volume.setUpdated(new Date()); + volume.setDomainId((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId()); + volume.setDisplayVolume(displayVolumeEnabled); + if (parentVolume != null) { + volume.setTemplateId(parentVolume.getTemplateId()); + volume.setFormat(parentVolume.getFormat()); + } else { + volume.setTemplateId(null); + } + + volume = _volsDao.persist(volume); + if (cmd.getSnapshotId() == null) { + // for volume created from snapshot, create usage event after volume creation + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId, + null, size, Volume.class.getName(), volume.getUuid()); + } + + CallContext.current().setEventDetails("Volume Id: " + volume.getId()); + + // Increment resource count during allocation; if actual creation fails, + // decrement it + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize())); + return volume; + } }); } @@ -957,6 +957,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic AsyncCallFuture future2 = volService.expungeVolumeAsync(volOnSecondary); future2.get(); } + // delete all cache entries for this volume + List cacheVols = volFactory.listVolumeOnCache(volume.getId()); + for (VolumeInfo volOnCache : cacheVols) { + s_logger.info("Delete volume from image cache store: " + volOnCache.getDataStore().getName()); + volOnCache.delete(); + } + } catch (Exception e) { s_logger.warn("Failed to expunge volume:", e); return false; diff --git a/server/src/com/cloud/template/HypervisorTemplateAdapter.java b/server/src/com/cloud/template/HypervisorTemplateAdapter.java index 00e62225b62..b33a19225ef 100755 --- a/server/src/com/cloud/template/HypervisorTemplateAdapter.java +++ b/server/src/com/cloud/template/HypervisorTemplateAdapter.java @@ -25,6 +25,8 @@ import java.util.concurrent.ExecutionException; import javax.ejb.Local; import javax.inject.Inject; +import org.apache.log4j.Logger; + import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd; import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd; import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd; @@ -44,7 +46,6 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback; import org.apache.cloudstack.framework.async.AsyncRpcContext; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; -import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.alert.AlertManager; @@ -56,11 +57,11 @@ import com.cloud.event.UsageEventUtils; import com.cloud.exception.InvalidParameterValueException; import com.cloud.exception.ResourceAllocationException; import com.cloud.org.Grouping; +import com.cloud.storage.ScopeType; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.TemplateProfile; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; -import com.cloud.storage.ScopeType; import com.cloud.storage.VMTemplateVO; import com.cloud.storage.VMTemplateZoneVO; import com.cloud.storage.dao.VMTemplateZoneDao; @@ -182,7 +183,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } // find all eligible image stores for this zone scope - List imageStores = this.storeMgr.getImageStoresByScope(new ZoneScope(profile.getZoneId())); + List imageStores = storeMgr.getImageStoresByScope(new ZoneScope(profile.getZoneId())); if ( imageStores == null || imageStores.size() == 0 ){ throw new CloudRuntimeException("Unable to find image store to download template "+ profile.getTemplate()); } @@ -205,12 +206,12 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } } - TemplateInfo tmpl = this.imageFactory.getTemplate(template.getId(), imageStore); + TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore); CreateTemplateContext context = new CreateTemplateContext(null, tmpl); AsyncCallbackDispatcher caller = AsyncCallbackDispatcher.create(this); caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null)); caller.setContext(context); - this.imageService.createTemplateAsync(tmpl, imageStore, caller); + imageService.createTemplateAsync(tmpl, imageStore, caller); if( !(profile.getIsPublic() || profile.getFeatured()) ){ // If private template then break break; } @@ -237,7 +238,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { // populated template entry _tmpltDao.remove(template.getId()); } else { - VMTemplateVO tmplt = this._tmpltDao.findById(template.getId()); + VMTemplateVO tmplt = _tmpltDao.findById(template.getId()); long accountId = tmplt.getAccountId(); if (template.getSize() != null) { // publish usage event @@ -283,7 +284,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { VMTemplateVO template = profile.getTemplate(); // find all eligible image stores for this template - List imageStores = this.templateMgr.getImageStoreByTemplate(template.getId(), profile.getZoneId()); + List imageStores = templateMgr.getImageStoreByTemplate(template.getId(), profile.getZoneId()); if (imageStores == null || imageStores.size() == 0) { // already destroyed on image stores s_logger.info("Unable to find image store still having template: " + template.getName() @@ -321,7 +322,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } s_logger.info("Delete template from image store: " + imageStore.getName()); - AsyncCallFuture future = this.imageService.deleteTemplateAsync(this.imageFactory + AsyncCallFuture future = imageService.deleteTemplateAsync(imageFactory .getTemplate(template.getId(), imageStore)); try { TemplateApiResult result = future.get(); @@ -350,9 +351,15 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { } } if (success) { + // delete all cache entries for this template + List cacheTmpls = imageFactory.listTemplateOnCache(template.getId()); + for (TemplateInfo tmplOnCache : cacheTmpls) { + s_logger.info("Delete template from image cache store: " + tmplOnCache.getDataStore().getName()); + tmplOnCache.delete(); + } // find all eligible image stores for this template - List iStores = this.templateMgr.getImageStoreByTemplate(template.getId(), null); + List iStores = templateMgr.getImageStoreByTemplate(template.getId(), null); if (iStores == null || iStores.size() == 0) { // remove template from vm_templates table if (_tmpltDao.remove(template.getId())) { @@ -380,7 +387,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { throw new InvalidParameterValueException("The DomR template cannot be deleted."); } - if (zoneId != null && (this.storeMgr.getImageStore(zoneId) == null)) { + if (zoneId != null && (storeMgr.getImageStore(zoneId) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone."); } @@ -392,7 +399,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase { TemplateProfile profile = super.prepareDelete(cmd); Long zoneId = profile.getZoneId(); - if (zoneId != null && (this.storeMgr.getImageStore(zoneId) == null)) { + if (zoneId != null && (storeMgr.getImageStore(zoneId) == null)) { throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone."); } diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 7a1b5aa5edc..d32776ad226 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -30,6 +30,8 @@ import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import org.apache.log4j.Logger; + import org.apache.cloudstack.acl.SecurityChecker.AccessType; import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd; import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd; @@ -50,7 +52,6 @@ import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd; import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; -import org.apache.cloudstack.engine.subsystem.api.storage.DataObject; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; @@ -81,7 +82,6 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; -import org.apache.log4j.Logger; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; @@ -481,9 +481,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("The " + desc + " has not been downloaded "); } - DataObject templateObject = _tmplFactory.getTemplate(templateId, tmpltStore); + // Handle NFS to S3 object store migration case, we trigger template sync from NFS to S3 during extract template or copy template + _tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore); - return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(), templateObject); + TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, tmpltStore); + + return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(), templateObject); } public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) { @@ -700,7 +703,15 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, throw new InvalidParameterValueException("Unable to find template with id"); } + DataStore srcSecStore = getImageStore(sourceZoneId, templateId); + if (srcSecStore == null) { + throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId); + } + if (template.isCrossZones()){ + //TODO: we may need UI still enable CopyTemplate in case of cross zone template to trigger sync to region store. + // sync template from cache store to region store if it is not there, for cases where we are going to migrate existing NFS to S3. + _tmpltSvr.syncTemplateToRegionStore(templateId, srcSecStore); s_logger.debug("Template " + templateId + " is cross-zone, don't need to copy"); return template; } @@ -712,11 +723,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return template; } - DataStore srcSecStore = getImageStore(sourceZoneId, templateId); - if (srcSecStore == null) { - throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId); - } - _accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template); boolean success = copy(userId, template, srcSecStore, dstZone); @@ -783,7 +789,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } try { - StoragePool pool = (StoragePool) this._dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId()); + StoragePool pool = (StoragePool) _dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId()); VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId()); if (s_logger.isDebugEnabled()) { @@ -1286,27 +1292,27 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Account owner = _accountMgr.getAccount(ownerId); final Domain domain = _domainDao.findById(owner.getDomainId()); if ("add".equalsIgnoreCase(operation)) { - final List accountNamesFinal = accountNames; + final List accountNamesFinal = accountNames; Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { for (String accountName : accountNamesFinal) { - Account permittedAccount = _accountDao.findActiveAccount(accountName, domain.getId()); - if (permittedAccount != null) { - if (permittedAccount.getId() == caller.getId()) { - continue; // don't grant permission to the template - // owner, they implicitly have permission - } - LaunchPermissionVO existingPermission = _launchPermissionDao.findByTemplateAndAccount(id, permittedAccount.getId()); - if (existingPermission == null) { - LaunchPermissionVO launchPermission = new LaunchPermissionVO(id, permittedAccount.getId()); - _launchPermissionDao.persist(launchPermission); - } - } else { - throw new InvalidParameterValueException("Unable to grant a launch permission to account " + accountName + " in domain id=" + domain.getUuid() - + ", account not found. " + "No permissions updated, please verify the account names and retry."); - } + Account permittedAccount = _accountDao.findActiveAccount(accountName, domain.getId()); + if (permittedAccount != null) { + if (permittedAccount.getId() == caller.getId()) { + continue; // don't grant permission to the template + // owner, they implicitly have permission } + LaunchPermissionVO existingPermission = _launchPermissionDao.findByTemplateAndAccount(id, permittedAccount.getId()); + if (existingPermission == null) { + LaunchPermissionVO launchPermission = new LaunchPermissionVO(id, permittedAccount.getId()); + _launchPermissionDao.persist(launchPermission); + } + } else { + throw new InvalidParameterValueException("Unable to grant a launch permission to account " + accountName + " in domain id=" + domain.getUuid() + + ", account not found. " + "No permissions updated, please verify the account names and retry."); + } + } } }); } else if ("remove".equalsIgnoreCase(operation)) { @@ -1436,23 +1442,23 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, Transaction.execute(new TransactionCallbackNoReturn() { @Override public void doInTransactionWithoutResult(TransactionStatus status) { - // template_store_ref entries should have been removed using our - // DataObject.processEvent command in case of failure, but clean - // it up here to avoid - // some leftovers which will cause removing template from - // vm_template table fail. - _tmplStoreDao.deletePrimaryRecordsForTemplate(templateId); - // Remove the template_zone_ref record - _tmpltZoneDao.deletePrimaryRecordsForTemplate(templateId); - // Remove the template record - _tmpltDao.expunge(templateId); - - // decrement resource count - if (accountId != null) { - _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); + // template_store_ref entries should have been removed using our + // DataObject.processEvent command in case of failure, but clean + // it up here to avoid + // some leftovers which will cause removing template from + // vm_template table fail. + _tmplStoreDao.deletePrimaryRecordsForTemplate(templateId); + // Remove the template_zone_ref record + _tmpltZoneDao.deletePrimaryRecordsForTemplate(templateId); + // Remove the template record + _tmpltDao.expunge(templateId); + + // decrement resource count + if (accountId != null) { + _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template); _resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(volumeFinal != null ? volumeFinal.getSize() : snapshotFinal.getSize())); - } + } } }); @@ -1714,6 +1720,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, return stores; } + @Override public VMTemplateVO updateTemplate(UpdateIsoCmd cmd) { return updateTemplateOrIso(cmd); @@ -1843,6 +1850,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, @Inject public void setTemplateAdapters(List adapters) { - this._adapters = adapters; + _adapters = adapters; } } diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 9c53a48ec52..aa73b2a74de 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -148,6 +148,7 @@ known_categories = { 'createSecondaryStagingStore': 'Image Store', 'deleteSecondaryStagingStore': 'Image Store', 'listSecondaryStagingStores': 'Image Store', + 'prepareSecondaryStorageForMigration' : 'Image Store', 'InternalLoadBalancer': 'Internal LB', 'DeploymentPlanners': 'Configuration', 'PortableIp': 'Portable IP',