etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-4024:Provide a way to upgrade from existing NFS secondary 

storage to S3.
This commit is contained in:
Min Chen 2013-10-28 21:01:31 -07:00
parent 271a7dff9e
commit 6be228a438
47 changed files with 1238 additions and 448 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/src/com/cloud/event/EventTypes.java
View File

@ -449,6 +449,9 @@ public class EventTypes {
public static final String EVENT_UCS_ASSOCIATED_PROFILE = "UCS.ASSOCIATEPROFILE";
// Object store migration
public static final String EVENT_MIGRATE_PREPARE_SECONDARY_STORAGE = "MIGRATE.PREPARE.SS";
static {
// TODO: need a way to force author adding event types to declare the entity details as well, with out braking

16
api/src/com/cloud/storage/StorageService.java
View File

@ -22,9 +22,9 @@ import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import com.cloud.exception.DiscoveryException;
@ -97,4 +97,18 @@ public interface StorageService{
ImageStore discoverImageStore(AddImageStoreCmd cmd) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
/**
* Prepare NFS secondary storage for object store migration
*
* @param cmd
* - the command specifying secondaryStorageId
* @return the storage pool
* @throws ResourceUnavailableException
* TODO
* @throws InsufficientCapacityException
* TODO
*/
public ImageStore prepareSecondaryStorageForObjectStoreMigration(Long storeId) throws ResourceUnavailableException,
InsufficientCapacityException;
}

1
api/src/org/apache/cloudstack/api/ApiCommandJobType.java
View File

@ -28,6 +28,7 @@ public enum ApiCommandJobType {
SystemVm,
Host,
StoragePool,
ImageStore,
IpAddress,
PortableIpAddress,
SecurityGroup,

109
api/src/org/apache/cloudstack/api/command/admin/storage/PrepareSecondaryStorageForMigrationCmd.java Normal file
View File

@ -0,0 +1,109 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.api.command.admin.storage;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.ApiCommandJobType;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.api.BaseAsyncCmd;
import org.apache.cloudstack.api.Parameter;
import org.apache.cloudstack.api.ServerApiException;
import org.apache.cloudstack.api.response.ImageStoreResponse;
import org.apache.cloudstack.context.CallContext;
import com.cloud.event.EventTypes;
import com.cloud.exception.InsufficientCapacityException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.storage.ImageStore;
import com.cloud.user.Account;
@APICommand(name = "prepareSecondaryStorageForMigration", description = "Prepare a NFS secondary storage to migrate to use object store like S3", responseObject = ImageStoreResponse.class)
public class PrepareSecondaryStorageForMigrationCmd extends BaseAsyncCmd {
public static final Logger s_logger = Logger.getLogger(PrepareSecondaryStorageForMigrationCmd.class.getName());
private static final String s_name = "preparesecondarystorageformigrationresponse";
/////////////////////////////////////////////////////
//////////////// API parameters /////////////////////
/////////////////////////////////////////////////////
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class,
required = true, description = "Secondary image store ID")
private Long id;
/////////////////////////////////////////////////////
/////////////////// Accessors ///////////////////////
/////////////////////////////////////////////////////
public Long getId() {
return id;
}
/////////////////////////////////////////////////////
/////////////// API Implementation///////////////////
/////////////////////////////////////////////////////
@Override
public String getCommandName() {
return s_name;
}
@Override
public ApiCommandJobType getInstanceType() {
return ApiCommandJobType.ImageStore;
}
@Override
public Long getInstanceId() {
return getId();
}
@Override
public long getEntityOwnerId() {
Account account = CallContext.current().getCallingAccount();
if (account != null) {
return account.getId();
}
return Account.ACCOUNT_ID_SYSTEM; // no account info given, parent this command to SYSTEM so ERROR events are tracked
}
@Override
public String getEventType() {
return EventTypes.EVENT_MIGRATE_PREPARE_SECONDARY_STORAGE;
}
@Override
public String getEventDescription() {
return "preparing secondary storage: " + getId() + " for object store migration";
}
@Override
public void execute() throws ResourceUnavailableException, InsufficientCapacityException{
ImageStore result = _storageService.prepareSecondaryStorageForObjectStoreMigration(getId());
if (result != null){
ImageStoreResponse response = _responseGenerator.createImageStoreResponse(result);
response.setResponseName(getCommandName());
response.setResponseName("secondarystorage");
setResponseObject(response);
} else {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to prepare secondary storage for object store migration");
}
}
}

1
client/tomcatconf/commands.properties.in
View File

@ -256,6 +256,7 @@ deleteImageStore=1
createSecondaryStagingStore=1
listSecondaryStagingStores=1
deleteSecondaryStagingStore=1
prepareSecondaryStorageForMigration=1
#### host commands
addHost=3

2
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreLifeCycle.java
View File

@ -37,4 +37,6 @@ public interface DataStoreLifeCycle {
boolean cancelMaintain(DataStore store);
boolean deleteDataStore(DataStore store);
boolean migrateToObjectStore(DataStore store);
}

3
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/DataStoreManager.java
View File

@ -19,6 +19,7 @@
package org.apache.cloudstack.engine.subsystem.api.storage;
import java.util.List;
import com.cloud.storage.DataStoreRole;
public interface DataStoreManager {
@ -37,4 +38,6 @@ public interface DataStoreManager {
DataStore getImageCacheStore(long zoneId);
List<DataStore> listImageStores();
List<DataStore> listImageCacheStores();
}

4
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotDataFactory.java
View File

@ -18,6 +18,8 @@
*/
package org.apache.cloudstack.engine.subsystem.api.storage;
import java.util.List;
import com.cloud.storage.DataStoreRole;
public interface SnapshotDataFactory {
@ -26,4 +28,6 @@ public interface SnapshotDataFactory {
SnapshotInfo getSnapshot(DataObject obj, DataStore store);
SnapshotInfo getSnapshot(long snapshotId, DataStoreRole role);
List<SnapshotInfo> listSnapshotOnCache(long snapshotId);
}

6
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateDataFactory.java
View File

@ -18,6 +18,8 @@
*/
package org.apache.cloudstack.engine.subsystem.api.storage;
import java.util.List;
import com.cloud.storage.DataStoreRole;
public interface TemplateDataFactory {
@ -28,4 +30,8 @@ public interface TemplateDataFactory {
TemplateInfo getTemplate(long templateId, DataStoreRole storeRole);
TemplateInfo getTemplate(long templateId, DataStoreRole storeRole, Long zoneId);
TemplateInfo getReadyTemplateOnCache(long templateId);
List<TemplateInfo> listTemplateOnCache(long templateId);
}

6
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/TemplateService.java
View File

@ -36,7 +36,7 @@ public interface TemplateService {
}
public TemplateInfo getTemplate() {
return this.template;
return template;
}
}
@ -54,6 +54,8 @@ public interface TemplateService {
AsyncCallFuture<TemplateApiResult> prepareTemplateOnPrimary(TemplateInfo srcTemplate, StoragePool pool);
void syncTemplateToRegionStore(long templateId, DataStore store);
void handleSysTemplateDownload(HypervisorType hostHyper, Long dcId);
void handleTemplateSync(DataStore store);
@ -62,5 +64,7 @@ public interface TemplateService {
void addSystemVMTemplatesToSecondary(DataStore store);
void associateTemplateToZone(long templateId, Long zoneId);
void associateCrosszoneTemplatesToZone(long dcId);
}

4
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeDataFactory.java
View File

@ -18,6 +18,8 @@
*/
package org.apache.cloudstack.engine.subsystem.api.storage;
import java.util.List;
import com.cloud.storage.DataStoreRole;
public interface VolumeDataFactory {
@ -28,4 +30,6 @@ public interface VolumeDataFactory {
VolumeInfo getVolume(long volumeId, DataStoreRole storeRole);
VolumeInfo getVolume(long volumeId);
List<VolumeInfo> listVolumeOnCache(long volumeId);
}

2
engine/schema/src/org/apache/cloudstack/storage/datastore/db/ImageStoreDao.java
View File

@ -36,4 +36,6 @@ public interface ImageStoreDao extends GenericDao<ImageStoreVO, Long> {
List<ImageStoreVO> findImageCacheByScope(ZoneScope scope);
List<ImageStoreVO> listImageStores();
List<ImageStoreVO> listImageCacheStores();
}

9
engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java
View File

@ -41,4 +41,13 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
List<SnapshotDataStoreVO> listDestroyed(long storeId);
List<SnapshotDataStoreVO> findBySnapshotId(long snapshotId);
void duplicateCacheRecordsOnRegionStore(long storeId);
// delete the snapshot entry on primary data store to make sure that next snapshot will be full snapshot
void deleteSnapshotRecordsOnPrimary();
List<SnapshotDataStoreVO> listOnCache(long snapshotId);
void updateStoreRoleToCache(long storeId);
}

34
engine/schema/src/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreVO.java
View File

@ -34,7 +34,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import com.cloud.storage.DataStoreRole;
import com.cloud.utils.db.GenericDao;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.fsm.StateObject;
@ -98,6 +97,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
@Column(name = "volume_id")
Long volumeId;
@Override
public String getInstallPath() {
return installPath;
}
@ -108,7 +108,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
}
public void setDataStoreId(long storeId) {
this.dataStoreId = storeId;
dataStoreId = storeId;
}
public long getSnapshotId() {
@ -135,15 +135,16 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
lastUpdated = date;
}
@Override
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
public SnapshotDataStoreVO(long hostId, long snapshotId) {
super();
this.dataStoreId = hostId;
dataStoreId = hostId;
this.snapshotId = snapshotId;
this.state = ObjectInDataStoreStateMachine.State.Allocated;
state = ObjectInDataStoreStateMachine.State.Allocated;
}
public SnapshotDataStoreVO() {
@ -158,14 +159,16 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
return jobId;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof SnapshotDataStoreVO) {
SnapshotDataStoreVO other = (SnapshotDataStoreVO) obj;
return (this.snapshotId == other.getSnapshotId() && this.dataStoreId == other.getDataStoreId());
return (snapshotId == other.getSnapshotId() && dataStoreId == other.getDataStoreId());
}
return false;
}
@Override
public int hashCode() {
Long tid = new Long(snapshotId);
Long hid = new Long(dataStoreId);
@ -192,21 +195,22 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
return -1;
}
@Override
public String toString() {
return new StringBuilder("SnapshotDataStore[").append(id).append("-").append(snapshotId).append("-")
.append(dataStoreId).append(installPath).append("]").toString();
}
public long getUpdatedCount() {
return this.updatedCount;
return updatedCount;
}
public void incrUpdatedCount() {
this.updatedCount++;
updatedCount++;
}
public void decrUpdatedCount() {
this.updatedCount--;
updatedCount--;
}
public Date getUpdated() {
@ -216,7 +220,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
@Override
public ObjectInDataStoreStateMachine.State getState() {
// TODO Auto-generated method stub
return this.state;
return state;
}
public void setState(ObjectInDataStoreStateMachine.State state) {
@ -225,7 +229,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
@Override
public long getObjectId() {
return this.getSnapshotId();
return getSnapshotId();
}
public DataStoreRole getRole() {
@ -238,7 +242,7 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
@Override
public State getObjectInStoreState() {
return this.state;
return state;
}
public long getParentSnapshotId() {
@ -253,12 +257,16 @@ public class SnapshotDataStoreVO implements StateObject<ObjectInDataStoreStateMa
return refCnt;
}
public void setRefCnt(Long refCnt) {
this.refCnt = refCnt;
}
public void incrRefCnt() {
this.refCnt++;
refCnt++;
}
public void decrRefCnt() {
this.refCnt--;
refCnt--;
}
public Long getVolumeId() {

10
engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java
View File

@ -18,9 +18,9 @@ package org.apache.cloudstack.storage.datastore.db;
import java.util.List;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
@ -62,4 +62,12 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
TemplateDataStoreVO findByTemplateZone(long templateId, Long zoneId, DataStoreRole role);
List<TemplateDataStoreVO> listByTemplate(long templateId);
void duplicateCacheRecordsOnRegionStore(long storeId);
TemplateDataStoreVO findReadyOnCache(long templateId);
List<TemplateDataStoreVO> listOnCache(long templateId);
void updateStoreRoleToCachce(long storeId);
}

51
engine/schema/src/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreVO.java
View File

@ -35,7 +35,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreState
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.fsm.StateObject;
@ -117,17 +116,17 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
public TemplateDataStoreVO(Long hostId, long templateId) {
super();
this.dataStoreId = hostId;
dataStoreId = hostId;
this.templateId = templateId;
this.state = ObjectInDataStoreStateMachine.State.Allocated;
this.refCnt = 0L;
state = ObjectInDataStoreStateMachine.State.Allocated;
refCnt = 0L;
}
public TemplateDataStoreVO(Long hostId, long templateId, Date lastUpdated, int downloadPercent,
Status downloadState, String localDownloadPath, String errorString, String jobId, String installPath,
String downloadUrl) {
super();
this.dataStoreId = hostId;
dataStoreId = hostId;
this.templateId = templateId;
this.lastUpdated = lastUpdated;
this.downloadPercent = downloadPercent;
@ -135,33 +134,33 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
this.localDownloadPath = localDownloadPath;
this.errorString = errorString;
this.jobId = jobId;
this.refCnt = 0L;
refCnt = 0L;
this.installPath = installPath;
this.setDownloadUrl(downloadUrl);
setDownloadUrl(downloadUrl);
switch (downloadState) {
case DOWNLOADED:
this.state = ObjectInDataStoreStateMachine.State.Ready;
state = ObjectInDataStoreStateMachine.State.Ready;
break;
case CREATING:
case DOWNLOAD_IN_PROGRESS:
case UPLOAD_IN_PROGRESS:
this.state = ObjectInDataStoreStateMachine.State.Creating2;
state = ObjectInDataStoreStateMachine.State.Creating2;
break;
case DOWNLOAD_ERROR:
case UPLOAD_ERROR:
this.state = ObjectInDataStoreStateMachine.State.Failed;
state = ObjectInDataStoreStateMachine.State.Failed;
break;
case ABANDONED:
this.state = ObjectInDataStoreStateMachine.State.Destroyed;
state = ObjectInDataStoreStateMachine.State.Destroyed;
break;
default:
this.state = ObjectInDataStoreStateMachine.State.Allocated;
state = ObjectInDataStoreStateMachine.State.Allocated;
break;
}
}
public TemplateDataStoreVO() {
this.refCnt = 0L;
refCnt = 0L;
}
@Override
@ -175,7 +174,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
}
public void setDataStoreId(long storeId) {
this.dataStoreId = storeId;
dataStoreId = storeId;
}
public long getTemplateId() {
@ -224,7 +223,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
}
public void setLocalDownloadPath(String localPath) {
this.localDownloadPath = localPath;
localDownloadPath = localPath;
}
public String getLocalDownloadPath() {
@ -251,7 +250,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
public boolean equals(Object obj) {
if (obj instanceof TemplateDataStoreVO) {
TemplateDataStoreVO other = (TemplateDataStoreVO) obj;
return (this.templateId == other.getTemplateId() && this.dataStoreId == other.getDataStoreId());
return (templateId == other.getTemplateId() && dataStoreId == other.getDataStoreId());
}
return false;
}
@ -316,7 +315,7 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
@Override
public ObjectInDataStoreStateMachine.State getState() {
// TODO Auto-generated method stub
return this.state;
return state;
}
public void setState(ObjectInDataStoreStateMachine.State state) {
@ -324,15 +323,15 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
}
public long getUpdatedCount() {
return this.updatedCount;
return updatedCount;
}
public void incrUpdatedCount() {
this.updatedCount++;
updatedCount++;
}
public void decrUpdatedCount() {
this.updatedCount--;
updatedCount--;
}
public Date getUpdated() {
@ -341,12 +340,12 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
@Override
public long getObjectId() {
return this.getTemplateId();
return getTemplateId();
}
@Override
public State getObjectInStoreState() {
return this.state;
return state;
}
public DataStoreRole getDataStoreRole() {
@ -361,12 +360,16 @@ public class TemplateDataStoreVO implements StateObject<ObjectInDataStoreStateMa
return refCnt;
}
public void setRefCnt(Long refCnt) {
this.refCnt = refCnt;
}
public void incrRefCnt() {
this.refCnt++;
refCnt++;
}
public void decrRefCnt() {
this.refCnt--;
refCnt--;
}
}

2
engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
View File

@ -40,4 +40,6 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
VolumeDataStoreVO findByStoreVolume(long storeId, long volumeId, boolean lock);
List<VolumeDataStoreVO> listDestroyed(long storeId);
void duplicateCacheRecordsOnRegionStore(long storeId);
}

47
engine/schema/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreVO.java
View File

@ -33,8 +33,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import com.cloud.storage.Storage;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.fsm.StateObject;
@ -117,6 +115,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
@Column(name = "ref_cnt")
Long refCnt = 0L;
@Override
public String getInstallPath() {
return installPath;
}
@ -127,7 +126,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
}
public void setDataStoreId(long storeId) {
this.dataStoreId = storeId;
dataStoreId = storeId;
}
public long getVolumeId() {
@ -174,6 +173,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
lastUpdated = date;
}
@Override
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
@ -192,17 +192,17 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
public VolumeDataStoreVO(long hostId, long volumeId) {
super();
this.dataStoreId = hostId;
dataStoreId = hostId;
this.volumeId = volumeId;
this.state = ObjectInDataStoreStateMachine.State.Allocated;
this.refCnt = 0L;
state = ObjectInDataStoreStateMachine.State.Allocated;
refCnt = 0L;
}
public VolumeDataStoreVO(long hostId, long volumeId, Date lastUpdated, int downloadPercent, Status downloadState,
String localDownloadPath, String errorString, String jobId, String installPath, String downloadUrl,
String checksum) {
// super();
this.dataStoreId = hostId;
dataStoreId = hostId;
this.volumeId = volumeId;
// this.zoneId = zoneId;
this.lastUpdated = lastUpdated;
@ -212,17 +212,17 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
this.errorString = errorString;
this.jobId = jobId;
this.installPath = installPath;
this.setDownloadUrl(downloadUrl);
setDownloadUrl(downloadUrl);
this.checksum = checksum;
this.refCnt = 0L;
refCnt = 0L;
}
public VolumeDataStoreVO() {
this.refCnt = 0L;
refCnt = 0L;
}
public void setLocalDownloadPath(String localPath) {
this.localDownloadPath = localPath;
localDownloadPath = localPath;
}
public String getLocalDownloadPath() {
@ -245,14 +245,16 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
return jobId;
}
@Override
public boolean equals(Object obj) {
if (obj instanceof VolumeDataStoreVO) {
VolumeDataStoreVO other = (VolumeDataStoreVO) obj;
return (this.volumeId == other.getVolumeId() && this.dataStoreId == other.getDataStoreId());
return (volumeId == other.getVolumeId() && dataStoreId == other.getDataStoreId());
}
return false;
}
@Override
public int hashCode() {
Long tid = new Long(volumeId);
Long hid = new Long(dataStoreId);
@ -295,21 +297,22 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
return -1;
}
@Override
public String toString() {
return new StringBuilder("VolumeDataStore[").append(id).append("-").append(volumeId).append("-").append(dataStoreId)
.append(installPath).append("]").toString();
}
public long getUpdatedCount() {
return this.updatedCount;
return updatedCount;
}
public void incrUpdatedCount() {
this.updatedCount++;
updatedCount++;
}
public void decrUpdatedCount() {
this.updatedCount--;
updatedCount--;
}
public Date getUpdated() {
@ -319,7 +322,7 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
@Override
public ObjectInDataStoreStateMachine.State getState() {
// TODO Auto-generated method stub
return this.state;
return state;
}
public void setState(ObjectInDataStoreStateMachine.State state) {
@ -328,24 +331,28 @@ public class VolumeDataStoreVO implements StateObject<ObjectInDataStoreStateMach
@Override
public long getObjectId() {
return this.getVolumeId();
return getVolumeId();
}
@Override
public State getObjectInStoreState() {
return this.state;
return state;
}
public Long getRefCnt() {
return refCnt;
}
public void setRefCnt(Long refCnt) {
this.refCnt = refCnt;
}
public void incrRefCnt() {
this.refCnt++;
refCnt++;
}
public void decrRefCnt() {
this.refCnt--;
refCnt--;
}
public String getExtractUrl() {

37
engine/storage/datamotion/src/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java
View File

@ -22,6 +22,9 @@ import java.util.Map;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
@ -48,9 +51,6 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.MigrateVolumeAnswer;
@ -65,7 +65,6 @@ import com.cloud.host.Host;
import com.cloud.host.dao.HostDao;
import com.cloud.server.ManagementService;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VolumeVO;
@ -112,8 +111,9 @@ AncientDataMotionStrategy implements DataMotionStrategy {
DataTO destTO = destData.getTO();
DataStoreTO srcStoreTO = srcTO.getDataStore();
DataStoreTO destStoreTO = destTO.getDataStore();
if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache ||
(srcStoreTO instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO)srcStoreTO).getPoolType() == StoragePoolType.NetworkFilesystem)) {
if (srcStoreTO instanceof NfsTO || srcStoreTO.getRole() == DataStoreRole.ImageCache) {
//||
// (srcStoreTO instanceof PrimaryDataStoreTO && ((PrimaryDataStoreTO)srcStoreTO).getPoolType() == StoragePoolType.NetworkFilesystem)) {
return false;
}
@ -185,7 +185,8 @@ AncientDataMotionStrategy implements DataMotionStrategy {
cacheMgr.deleteCacheObject(srcForCopy);
} else {
// for template, we want to leave it on cache for performance reason
if (answer == null || !answer.getResult()) {
if ((answer == null || !answer.getResult()) && srcForCopy.getRefCount() < 2) {
// cache object created by this copy, not already there
cacheMgr.deleteCacheObject(srcForCopy);
} else {
cacheMgr.releaseCacheObject(srcForCopy);
@ -222,6 +223,13 @@ AncientDataMotionStrategy implements DataMotionStrategy {
}
}
protected void releaseSnapshotCacheChain(SnapshotInfo snapshot) {
while (snapshot != null) {
cacheMgr.releaseCacheObject(snapshot);
snapshot = snapshot.getParent();
}
}
protected Answer copyVolumeFromSnapshot(DataObject snapObj, DataObject volObj) {
SnapshotInfo snapshot = (SnapshotInfo) snapObj;
StoragePool pool = (StoragePool) volObj.getDataStore();
@ -255,7 +263,8 @@ AncientDataMotionStrategy implements DataMotionStrategy {
throw new CloudRuntimeException(basicErrMsg);
} finally {
if (!(storTO instanceof NfsTO)) {
deleteSnapshotCacheChain((SnapshotInfo) srcData);
// still keep snapshot on cache which may be migrated from previous secondary storage
releaseSnapshotCacheChain((SnapshotInfo)srcData);
}
}
}
@ -281,7 +290,7 @@ AncientDataMotionStrategy implements DataMotionStrategy {
if (cacheStore == null) {
// need to find a nfs or cifs image store, assuming that can't copy volume
// directly to s3
ImageStoreEntity imageStore = (ImageStoreEntity) this.dataStoreMgr.getImageStore(destScope.getScopeId());
ImageStoreEntity imageStore = (ImageStoreEntity) dataStoreMgr.getImageStore(destScope.getScopeId());
if (!imageStore.getProtocol().equalsIgnoreCase("nfs") && !imageStore.getProtocol().equalsIgnoreCase("cifs")) {
s_logger.debug("can't find a nfs (or cifs) image store to satisfy the need for a staging store");
return null;
@ -290,7 +299,7 @@ AncientDataMotionStrategy implements DataMotionStrategy {
DataObject objOnImageStore = imageStore.create(srcData);
objOnImageStore.processEvent(Event.CreateOnlyRequested);
Answer answer = this.copyObject(srcData, objOnImageStore);
Answer answer = copyObject(srcData, objOnImageStore);
if (answer == null || !answer.getResult()) {
if (answer != null) {
s_logger.debug("copy to image store failed: " + answer.getDetails());
@ -336,7 +345,7 @@ AncientDataMotionStrategy implements DataMotionStrategy {
protected Answer migrateVolumeToPool(DataObject srcData, DataObject destData) {
VolumeInfo volume = (VolumeInfo)srcData;
StoragePool destPool = (StoragePool)this.dataStoreMgr.getDataStore(destData.getDataStore().getId(), DataStoreRole.Primary);
StoragePool destPool = (StoragePool)dataStoreMgr.getDataStore(destData.getDataStore().getId(), DataStoreRole.Primary);
MigrateVolumeCommand command = new MigrateVolumeCommand(volume.getId(), volume.getPath(), destPool);
EndPoint ep = selector.select(volume.getDataStore());
MigrateVolumeAnswer answer = (MigrateVolumeAnswer) ep.sendMessage(command);
@ -345,14 +354,14 @@ AncientDataMotionStrategy implements DataMotionStrategy {
throw new CloudRuntimeException("Failed to migrate volume " + volume + " to storage pool " + destPool);
} else {
// Update the volume details after migration.
VolumeVO volumeVo = this.volDao.findById(volume.getId());
VolumeVO volumeVo = volDao.findById(volume.getId());
Long oldPoolId = volume.getPoolId();
volumeVo.setPath(answer.getVolumePath());
volumeVo.setFolder(destPool.getPath());
volumeVo.setPodId(destPool.getPodId());
volumeVo.setPoolId(destPool.getId());
volumeVo.setLastPoolId(oldPoolId);
this.volDao.update(volume.getId(), volumeVo);
volDao.update(volume.getId(), volumeVo);
}
return answer;
@ -426,7 +435,7 @@ AncientDataMotionStrategy implements DataMotionStrategy {
// clean up snapshot copied to staging
if (needCache && srcData != null) {
cacheMgr.deleteCacheObject(srcData);
cacheMgr.releaseCacheObject(srcData); // reduce ref count, but keep it there on cache which is converted from previous secondary storage
}
return answer;
}

38
engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateDataFactoryImpl.java
View File

@ -18,8 +18,14 @@
*/
package org.apache.cloudstack.storage.image;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -28,8 +34,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VMTemplateStoragePoolVO;
@ -87,7 +91,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
TemplateDataStoreVO tmplStore = templateStoreDao.findByTemplate(templateId, storeRole);
DataStore store = null;
if (tmplStore != null) {
store = this.storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
store = storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
}
return this.getTemplate(templateId, store);
}
@ -97,7 +101,7 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
TemplateDataStoreVO tmplStore = templateStoreDao.findByTemplateZone(templateId, zoneId, storeRole);
DataStore store = null;
if (tmplStore != null) {
store = this.storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
store = storeMgr.getDataStore(tmplStore.getDataStoreId(), storeRole);
}
return this.getTemplate(templateId, store);
}
@ -113,4 +117,30 @@ public class TemplateDataFactoryImpl implements TemplateDataFactory {
tmpObj.setUrl(origTmpl.getUrl());
return tmpObj;
}
@Override
public TemplateInfo getReadyTemplateOnCache(long templateId) {
TemplateDataStoreVO tmplStore = templateStoreDao.findReadyOnCache(templateId);
if (tmplStore != null) {
DataStore store = storeMgr.getDataStore(tmplStore.getDataStoreId(), DataStoreRole.ImageCache);
return getTemplate(templateId, store);
} else {
return null;
}
}
@Override
public List<TemplateInfo> listTemplateOnCache(long templateId) {
List<TemplateDataStoreVO> cacheTmpls = templateStoreDao.listOnCache(templateId);
List<TemplateInfo> tmplObjs = new ArrayList<TemplateInfo>();
for (TemplateDataStoreVO cacheTmpl : cacheTmpls) {
long storeId = cacheTmpl.getDataStoreId();
DataStore store = storeMgr.getDataStore(storeId, DataStoreRole.ImageCache);
TemplateInfo tmplObj = getTemplate(templateId, store);
tmplObjs.add(tmplObj);
}
return tmplObjs;
}
}

95
engine/storage/image/src/org/apache/cloudstack/storage/image/TemplateServiceImpl.java
View File

@ -28,6 +28,9 @@ import java.util.Set;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.CopyCommandResult;
import org.apache.cloudstack.engine.subsystem.api.storage.CreateCmdResult;
import org.apache.cloudstack.engine.subsystem.api.storage.DataMotionService;
@ -39,6 +42,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.StorageCacheManager;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
@ -59,9 +63,6 @@ import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.storage.ListTemplateAnswer;
import com.cloud.agent.api.storage.ListTemplateCommand;
@ -73,8 +74,9 @@ import com.cloud.dc.dao.DataCenterDao;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.StoragePool;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VMTemplateZoneVO;
@ -128,6 +130,8 @@ public class TemplateServiceImpl implements TemplateService {
TemplateManager _tmpltMgr;
@Inject
ConfigurationDao _configDao;
@Inject
StorageCacheManager _cacheMgr;
class TemplateOpContext<T> extends AsyncRpcContext<T> {
final TemplateObject template;
@ -466,7 +470,8 @@ public class TemplateServiceImpl implements TemplateService {
// persist entry in template_zone_ref table. zoneId can be empty for
// region-wide image store, in that case,
// we will associate the template to all the zones.
private void associateTemplateToZone(long templateId, Long zoneId) {
@Override
public void associateTemplateToZone(long templateId, Long zoneId) {
List<Long> dcs = new ArrayList<Long>();
if (zoneId != null) {
dcs.add(zoneId);
@ -608,6 +613,86 @@ public class TemplateServiceImpl implements TemplateService {
return copyAsync(volume, template, store);
}
private AsyncCallFuture<TemplateApiResult> syncToRegionStoreAsync(TemplateInfo template, DataStore store) {
AsyncCallFuture<TemplateApiResult> future = new AsyncCallFuture<TemplateApiResult>();
// no need to create entry on template_store_ref here, since entries are already created when prepareSecondaryStorageForMigration is invoked.
// But we need to set default install path so that sync can be done in the right s3 path
TemplateInfo templateOnStore = _templateFactory.getTemplate(template, store);
String installPath = TemplateConstants.DEFAULT_TMPLT_ROOT_DIR + "/"
+ TemplateConstants.DEFAULT_TMPLT_FIRST_LEVEL_DIR
+ template.getAccountId() + "/" + template.getId() + "/" + template.getUniqueName();
((TemplateObject)templateOnStore).setInstallPath(installPath);
TemplateOpContext<TemplateApiResult> context = new TemplateOpContext<TemplateApiResult>(null,
(TemplateObject)templateOnStore, future);
AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult> caller = AsyncCallbackDispatcher.create(this);
caller.setCallback(caller.getTarget().syncTemplateCallBack(null, null)).setContext(context);
_motionSrv.copyAsync(template, templateOnStore, caller);
return future;
}
protected Void syncTemplateCallBack(AsyncCallbackDispatcher<TemplateServiceImpl, CopyCommandResult> callback,
TemplateOpContext<TemplateApiResult> context) {
TemplateInfo destTemplate = context.getTemplate();
CopyCommandResult result = callback.getResult();
AsyncCallFuture<TemplateApiResult> future = context.getFuture();
TemplateApiResult res = new TemplateApiResult(destTemplate);
try {
if (result.isFailed()) {
res.setResult(result.getResult());
// no change to existing template_store_ref, will try to re-sync later if other call triggers this sync operation, like copy template
} else {
// this will update install path properly, next time it will not sync anymore.
destTemplate.processEvent(Event.OperationSuccessed, result.getAnswer());
}
future.complete(res);
} catch (Exception e) {
s_logger.debug("Failed to process sync template callback", e);
res.setResult(e.toString());
future.complete(res);
}
return null;
}
private boolean isRegionStore(DataStore store) {
if (store.getScope().getScopeType() == ScopeType.ZONE && store.getScope().getScopeId() == null)
return true;
else
return false;
}
// This routine is used to push templates currently on cache store, but not in region store to region store.
// used in migrating existing NFS secondary storage to S3.
@Override
public void syncTemplateToRegionStore(long templateId, DataStore store) {
if (isRegionStore(store)) {
// if template is on region wide object store, check if it is really downloaded there (by checking install_path). Sync template to region
// wide store if it is not there physically.
TemplateInfo tmplOnStore = _templateFactory.getTemplate(templateId, store);
if (tmplOnStore == null) {
throw new CloudRuntimeException("Cannot find an entry in template_store_ref for template " + templateId + " on region store: " + store.getName());
}
if (tmplOnStore.getInstallPath() == null || tmplOnStore.getInstallPath().length() == 0) {
// template is not on region store yet, sync to region store
TemplateInfo srcTemplate = _templateFactory.getReadyTemplateOnCache(templateId);
if (srcTemplate == null) {
throw new CloudRuntimeException("Cannot find template " + templateId + " on cache store");
}
AsyncCallFuture<TemplateApiResult> future = syncToRegionStoreAsync(srcTemplate, store);
try {
TemplateApiResult result = future.get();
if (result.isFailed()) {
throw new CloudRuntimeException("sync template from cache to region wide store failed for image store " + store.getName() + ":"
+ result.getResult());
}
_cacheMgr.releaseCacheObject(srcTemplate); // reduce reference count for template on cache, so it can recycled by schedule
} catch (Exception ex) {
throw new CloudRuntimeException("sync template from cache to region wide store failed for image store " + store.getName());
}
}
}
}
@Override
public AsyncCallFuture<TemplateApiResult> copyTemplate(TemplateInfo srcTemplate, DataStore destStore) {
// generate a URL from source template ssvm to download to destination data store

15
engine/storage/image/src/org/apache/cloudstack/storage/image/manager/ImageStoreProviderManagerImpl.java
View File

@ -26,6 +26,9 @@ import java.util.Map;
import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider;
@ -37,8 +40,6 @@ import org.apache.cloudstack.storage.image.ImageStoreDriver;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
import org.apache.cloudstack.storage.image.store.ImageStoreImpl;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.storage.ScopeType;
import com.cloud.storage.dao.VMTemplateDao;
@ -94,6 +95,16 @@ public class ImageStoreProviderManagerImpl implements ImageStoreProviderManager
return imageStores;
}
@Override
public List<DataStore> listImageCacheStores() {
List<ImageStoreVO> stores = dataStoreDao.listImageCacheStores();
List<DataStore> imageStores = new ArrayList<DataStore>();
for (ImageStoreVO store : stores) {
imageStores.add(getImageStore(store.getId()));
}
return imageStores;
}
@Override
public List<DataStore> listImageStoresByScope(ZoneScope scope) {
List<ImageStoreVO> stores = dataStoreDao.findByScope(scope);

8
engine/storage/image/src/org/apache/cloudstack/storage/image/store/TemplateObject.java
View File

@ -58,6 +58,7 @@ public class TemplateObject implements TemplateInfo {
private VMTemplateVO imageVO;
private DataStore dataStore;
private String url;
private String installPath; // temporarily set installPath before passing to resource for entries with empty installPath for object store migration case
@Inject
VMTemplateDao imageDao;
@Inject
@ -293,6 +294,9 @@ public class TemplateObject implements TemplateInfo {
@Override
public String getInstallPath() {
if (installPath != null)
return installPath;
if (dataStore == null) {
return null;
}
@ -300,6 +304,10 @@ public class TemplateObject implements TemplateInfo {
return obj.getInstallPath();
}
public void setInstallPath(String installPath) {
this.installPath = installPath;
}
@Override
public long getAccountId() {
return imageVO.getAccountId();

22
engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java
View File

@ -18,8 +18,13 @@
*/
package org.apache.cloudstack.storage.snapshot;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -28,7 +33,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.springframework.stereotype.Component;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.SnapshotVO;
@ -70,8 +74,22 @@ public class SnapshotDataFactoryImpl implements SnapshotDataFactory {
if (snapshotStore == null) {
return null;
}
DataStore store = this.storeMgr.getDataStore(snapshotStore.getDataStoreId(), role);
DataStore store = storeMgr.getDataStore(snapshotStore.getDataStoreId(), role);
SnapshotObject so = SnapshotObject.getSnapshotObject(snapshot, store);
return so;
}
@Override
public List<SnapshotInfo> listSnapshotOnCache(long snapshotId) {
List<SnapshotDataStoreVO> cacheSnapshots = snapshotStoreDao.listOnCache(snapshotId);
List<SnapshotInfo> snapObjs = new ArrayList<SnapshotInfo>();
for (SnapshotDataStoreVO cacheSnap : cacheSnapshots) {
long storeId = cacheSnap.getDataStoreId();
DataStore store = storeMgr.getDataStore(storeId, DataStoreRole.ImageCache);
SnapshotInfo tmplObj = getSnapshot(snapshotId, store);
snapObjs.add(tmplObj);
}
return snapObjs;
}
}

57
engine/storage/snapshot/src/org/apache/cloudstack/storage/snapshot/XenserverSnapshotStrategy.java
View File

@ -16,17 +16,14 @@
// under the License.
package org.apache.cloudstack.storage.snapshot;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Snapshot;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Volume;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.DB;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
import java.util.List;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
@ -42,10 +39,18 @@ import org.apache.cloudstack.storage.command.CreateObjectAnswer;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Snapshot;
import com.cloud.storage.SnapshotVO;
import com.cloud.storage.Volume;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.utils.NumbersUtil;
import com.cloud.utils.db.DB;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
@Component
public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
@ -71,7 +76,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
if (parentSnapshot != null && snapshot.getPath().equalsIgnoreCase(parentSnapshot.getPath())) {
s_logger.debug("backup an empty snapshot");
// don't need to backup this snapshot
SnapshotDataStoreVO parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(
SnapshotDataStoreVO parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(
parentSnapshot.getId(), DataStoreRole.Image);
if (parentSnapshotOnBackupStore != null && parentSnapshotOnBackupStore.getState() == State.Ready) {
DataStore store = dataStoreMgr.getDataStore(parentSnapshotOnBackupStore.getDataStoreId(),
@ -93,7 +98,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
s_logger.debug("Failed to change state: " + snapshot.getId() + ": " + e.toString());
throw new CloudRuntimeException(e.toString());
}
return this.snapshotDataFactory.getSnapshot(snapObj.getId(), store);
return snapshotDataFactory.getSnapshot(snapObj.getId(), store);
} else {
s_logger.debug("parent snapshot hasn't been backed up yet");
}
@ -111,7 +116,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
int i;
SnapshotDataStoreVO parentSnapshotOnBackupStore = null;
for (i = 1; i < deltaSnap; i++) {
parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(parentSnapshot.getId(),
parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(parentSnapshot.getId(),
DataStoreRole.Image);
if (parentSnapshotOnBackupStore == null) {
break;
@ -122,7 +127,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
break;
}
parentSnapshotOnBackupStore = this.snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image);
parentSnapshotOnBackupStore = snapshotStoreDao.findBySnapshot(prevBackupId, DataStoreRole.Image);
}
if (i >= deltaSnap) {
fullBackup = true;
@ -130,7 +135,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
}
snapshot.addPayload(fullBackup);
return this.snapshotSvr.backupSnapshot(snapshot);
return snapshotSvr.backupSnapshot(snapshot);
}
protected boolean deleteSnapshotChain(SnapshotInfo snapshot) {
@ -164,7 +169,15 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
}
}
if (!deleted) {
boolean r = this.snapshotSvr.deleteSnapshot(snapshot);
boolean r = snapshotSvr.deleteSnapshot(snapshot);
if (r) {
// delete snapshot in cache if there is
List<SnapshotInfo> cacheSnaps = snapshotDataFactory.listSnapshotOnCache(snapshot.getId());
for (SnapshotInfo cacheSnap : cacheSnaps) {
s_logger.debug("Delete snapshot " + snapshot.getId() + " from image cache store: " + cacheSnap.getDataStore().getName());
cacheSnap.delete();
}
}
if (!resultIsSet) {
result = r;
resultIsSet = true;
@ -200,7 +213,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
// first mark the snapshot as destroyed, so that ui can't see it, but we
// may not destroy the snapshot on the storage, as other snapshots may
// depend on it.
SnapshotInfo snapshotOnImage = this.snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image);
SnapshotInfo snapshotOnImage = snapshotDataFactory.getSnapshot(snapshotId, DataStoreRole.Image);
if (snapshotOnImage == null) {
s_logger.debug("Can't find snapshot on backup storage, delete it in db");
snapshotDao.remove(snapshotId);
@ -273,7 +286,7 @@ public class XenserverSnapshotStrategy extends SnapshotStrategyBase {
snapshot = result.getSnashot();
DataStore primaryStore = snapshot.getDataStore();
SnapshotInfo backupedSnapshot = this.backupSnapshot(snapshot);
SnapshotInfo backupedSnapshot = backupSnapshot(snapshot);
try {
SnapshotInfo parent = snapshot.getParent();

5
engine/storage/src/org/apache/cloudstack/storage/datastore/DataStoreManagerImpl.java
View File

@ -107,6 +107,11 @@ public class DataStoreManagerImpl implements DataStoreManager {
return imageDataStoreMgr.listImageStores();
}
@Override
public List<DataStore> listImageCacheStores() {
return imageDataStoreMgr.listImageCacheStores();
}
public void setPrimaryStoreMgr(PrimaryDataStoreProviderManager primaryStoreMgr) {
this.primaryStoreMgr = primaryStoreMgr;
}

21
engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreHelper.java
View File

@ -24,11 +24,14 @@ import java.util.UUID;
import javax.inject.Inject;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailVO;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
@ -40,6 +43,8 @@ public class ImageStoreHelper {
ImageStoreDao imageStoreDao;
@Inject
ImageStoreDetailsDao imageStoreDetailsDao;
@Inject
SnapshotDataStoreDao snapshotStoreDao;
public ImageStoreVO createImageStore(Map<String, Object> params) {
ImageStoreVO store = imageStoreDao.findByName((String) params.get("name"));
@ -115,4 +120,18 @@ public class ImageStoreHelper {
imageStoreDao.remove(id);
return true;
}
/**
* Convert current NFS secondary storage to Staging store to be ready to migrate to S3 object store.
* @param store NFS image store.
* @return true if successful.
*/
public boolean convertToStagingStore(DataStore store) {
ImageStoreVO nfsStore = imageStoreDao.findById(store.getId());
nfsStore.setRole(DataStoreRole.ImageCache);
imageStoreDao.update(store.getId(), nfsStore);
// clear snapshot entry on primary store to make next snapshot become full snapshot
snapshotStoreDao.deleteSnapshotRecordsOnPrimary();
return true;
}
}

2
engine/storage/src/org/apache/cloudstack/storage/image/datastore/ImageStoreProviderManager.java
View File

@ -32,6 +32,8 @@ public interface ImageStoreProviderManager {
List<DataStore> listImageStores();
List<DataStore> listImageCacheStores();
List<DataStore> listImageStoresByScope(ZoneScope scope);
List<DataStore> listImageStoreByProvider(String provider);

10
engine/storage/src/org/apache/cloudstack/storage/image/db/ImageStoreDaoImpl.java
View File

@ -23,10 +23,11 @@ import java.util.Map;
import javax.naming.ConfigurationException;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.springframework.stereotype.Component;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
@ -118,4 +119,11 @@ public class ImageStoreDaoImpl extends GenericDaoBase<ImageStoreVO, Long> implem
return listBy(sc);
}
@Override
public List<ImageStoreVO> listImageCacheStores() {
SearchCriteria<ImageStoreVO> sc = createSearchCriteria();
sc.addAnd("role", SearchCriteria.Op.EQ, DataStoreRole.ImageCache);
return listBy(sc);
}
}

82
engine/storage/src/org/apache/cloudstack/storage/image/db/SnapshotDataStoreDaoImpl.java
View File

@ -25,23 +25,23 @@ import java.util.Map;
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.utils.db.DB;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.storage.DataStoreRole;
import com.cloud.utils.db.DB;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.UpdateBuilder;
@Component
@ -54,7 +54,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
private SearchBuilder<SnapshotDataStoreVO> snapshotSearch;
private SearchBuilder<SnapshotDataStoreVO> storeSnapshotSearch;
private SearchBuilder<SnapshotDataStoreVO> snapshotIdSearch;
private String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " +
private final String parentSearch = "select store_id, store_role, snapshot_id from cloud.snapshot_store_ref where store_id = ? " +
" and store_role = ? and volume_id = ? and state = 'Ready'" +
" order by created DESC " +
" limit 1";
@ -170,6 +170,16 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
txn.commit();
}
@Override
public void deleteSnapshotRecordsOnPrimary() {
SearchCriteria<SnapshotDataStoreVO> sc = storeSearch.create();
sc.setParameters("store_role", DataStoreRole.Primary);
TransactionLegacy txn = TransactionLegacy.currentTxn();
txn.start();
remove(sc);
txn.commit();
}
@Override
public SnapshotDataStoreVO findByStoreSnapshot(DataStoreRole role, long storeId, long snapshotId) {
SearchCriteria<SnapshotDataStoreVO> sc = storeSnapshotSearch.create();
@ -195,7 +205,7 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
long sid = rs.getLong(1);
String rl = rs.getString(2);
long snid = rs.getLong(3);
return this.findByStoreSnapshot(role, sid, snid);
return findByStoreSnapshot(role, sid, snid);
}
} catch (SQLException e) {
s_logger.debug("Failed to find parent snapshot: " + e.toString());
@ -236,4 +246,64 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase<SnapshotDataStoreVO
sc.setParameters("ref_cnt", 0);
return listBy(sc);
}
@Override
public void duplicateCacheRecordsOnRegionStore(long storeId) {
// find all records on image cache
SearchCriteria<SnapshotDataStoreVO> sc = storeSnapshotSearch.create();
sc.setParameters("store_role", DataStoreRole.ImageCache);
sc.setParameters("destroyed", false);
List<SnapshotDataStoreVO> snapshots = listBy(sc);
// create an entry for each record, but with empty install path since the content is not yet on region-wide store yet
if (snapshots != null) {
s_logger.info("Duplicate " + snapshots.size() + " snapshot cache store records to region store");
for (SnapshotDataStoreVO snap : snapshots) {
SnapshotDataStoreVO snapStore = findByStoreSnapshot(DataStoreRole.Image, storeId, snap.getSnapshotId());
if (snapStore != null) {
s_logger.info("There is already entry for snapshot " + snap.getSnapshotId() + " on region store " + storeId);
continue;
}
s_logger.info("Persisting an entry for snapshot " + snap.getSnapshotId() + " on region store " + storeId);
SnapshotDataStoreVO ss = new SnapshotDataStoreVO();
ss.setSnapshotId(snap.getSnapshotId());
ss.setDataStoreId(storeId);
ss.setRole(DataStoreRole.Image);
ss.setVolumeId(snap.getVolumeId());
ss.setParentSnapshotId(snap.getParentSnapshotId());
ss.setState(snap.getState());
ss.setSize(snap.getSize());
ss.setPhysicalSize(snap.getPhysicalSize());
ss.setRefCnt(snap.getRefCnt());
persist(ss);
// increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
snap.incrRefCnt();
update(snap.getId(), snap);
}
}
}
@Override
public List<SnapshotDataStoreVO> listOnCache(long snapshotId) {
SearchCriteria<SnapshotDataStoreVO> sc = storeSnapshotSearch.create();
sc.setParameters("snapshot_id", snapshotId);
sc.setParameters("store_role", DataStoreRole.ImageCache);
return search(sc, null);
}
@Override
public void updateStoreRoleToCache(long storeId) {
SearchCriteria<SnapshotDataStoreVO> sc = storeSearch.create();
sc.setParameters("store_id", storeId);
sc.setParameters("destroyed", false);
List<SnapshotDataStoreVO> snaps = listBy(sc);
if (snaps != null) {
s_logger.info("Update to cache store role for " + snaps.size() + " entries in snapshot_store_ref");
for (SnapshotDataStoreVO snap : snaps) {
snap.setRole(DataStoreRole.ImageCache);
update(snap.getId(), snap);
}
}
}
}

104
engine/storage/src/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java
View File

@ -31,20 +31,26 @@ import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.UpdateBuilder;
import com.cloud.utils.exception.CloudRuntimeException;
@Component
public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO, Long> implements TemplateDataStoreDao {
@ -61,6 +67,11 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
@Inject
private DataStoreManager _storeMgr;
@Inject
private VMTemplateDao _tmpltDao;
@Inject
private TemplateService _tmplSrv;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
super.configure(name, params);
@ -85,6 +96,7 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
templateRoleSearch.and("template_id", templateRoleSearch.entity().getTemplateId(), SearchCriteria.Op.EQ);
templateRoleSearch.and("store_role", templateRoleSearch.entity().getDataStoreRole(), SearchCriteria.Op.EQ);
templateRoleSearch.and("destroyed", templateRoleSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
templateRoleSearch.and("state", templateRoleSearch.entity().getState(), SearchCriteria.Op.EQ);
templateRoleSearch.done();
updateStateSearch = this.createSearchBuilder();
@ -313,6 +325,24 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
return findOneIncludingRemovedBy(sc);
}
@Override
public TemplateDataStoreVO findReadyOnCache(long templateId) {
SearchCriteria<TemplateDataStoreVO> sc = templateRoleSearch.create();
sc.setParameters("template_id", templateId);
sc.setParameters("store_role", DataStoreRole.ImageCache);
sc.setParameters("destroyed", false);
sc.setParameters("state", ObjectInDataStoreStateMachine.State.Ready);
return findOneIncludingRemovedBy(sc);
}
@Override
public List<TemplateDataStoreVO> listOnCache(long templateId) {
SearchCriteria<TemplateDataStoreVO> sc = templateRoleSearch.create();
sc.setParameters("template_id", templateId);
sc.setParameters("store_role", DataStoreRole.ImageCache);
return search(sc, null);
}
@Override
public List<TemplateDataStoreVO> listByTemplate(long templateId) {
SearchCriteria<TemplateDataStoreVO> sc = templateSearch.create();
@ -341,4 +371,78 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase<TemplateDataStoreVO
return null;
}
/**
* Duplicate all image cache store entries
*/
@Override
public void duplicateCacheRecordsOnRegionStore(long storeId) {
// find all records on image cache
SearchCriteria<TemplateDataStoreVO> sc = templateRoleSearch.create();
sc.setParameters("store_role", DataStoreRole.ImageCache);
sc.setParameters("destroyed", false);
List<TemplateDataStoreVO> tmpls = listBy(sc);
// create an entry for each template record, but with empty install path since the content is not yet on region-wide store yet
if (tmpls != null) {
s_logger.info("Duplicate " + tmpls.size() + " template cache store records to region store");
for (TemplateDataStoreVO tmpl : tmpls) {
long templateId = tmpl.getTemplateId();
VMTemplateVO template = _tmpltDao.findById(templateId);
if (template == null) {
throw new CloudRuntimeException("No template is found for template id: " + templateId);
}
if (template.getTemplateType() == TemplateType.SYSTEM) {
s_logger.info("No need to duplicate system template since it will be automatically downloaded while adding region store");
continue;
}
TemplateDataStoreVO tmpStore = findByStoreTemplate(storeId, tmpl.getTemplateId());
if (tmpStore != null) {
s_logger.info("There is already entry for template " + tmpl.getTemplateId() + " on region store " + storeId);
continue;
}
s_logger.info("Persisting an entry for template " + tmpl.getTemplateId() + " on region store " + storeId);
TemplateDataStoreVO ts = new TemplateDataStoreVO();
ts.setTemplateId(tmpl.getTemplateId());
ts.setDataStoreId(storeId);
ts.setDataStoreRole(DataStoreRole.Image);
ts.setState(tmpl.getState());
ts.setDownloadPercent(tmpl.getDownloadPercent());
ts.setDownloadState(tmpl.getDownloadState());
ts.setSize(tmpl.getSize());
ts.setPhysicalSize(tmpl.getPhysicalSize());
ts.setErrorString(tmpl.getErrorString());
ts.setDownloadUrl(tmpl.getDownloadUrl());
ts.setRefCnt(tmpl.getRefCnt());
persist(ts);
// increase ref_cnt of cache store entry so that this will not be recycled before the content is pushed to region-wide store
tmpl.incrRefCnt();
this.update(tmpl.getId(), tmpl);
// mark the template as cross-zones
template.setCrossZones(true);
_tmpltDao.update(templateId, template);
// add template_zone_ref association for these cross-zone templates
_tmplSrv.associateTemplateToZone(templateId, null);
}
}
}
@Override
public void updateStoreRoleToCachce(long storeId) {
SearchCriteria<TemplateDataStoreVO> sc = storeSearch.create();
sc.setParameters("store_id", storeId);
sc.setParameters("destroyed", false);
List<TemplateDataStoreVO> tmpls = listBy(sc);
if (tmpls != null) {
s_logger.info("Update to cache store role for " + tmpls.size() + " entries in template_store_ref");
for (TemplateDataStoreVO tmpl : tmpls) {
tmpl.setDataStoreRole(DataStoreRole.ImageCache);
update(tmpl.getId(), tmpl);
}
}
}
}

55
engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
View File

@ -16,25 +16,30 @@
// under the License.
package org.apache.cloudstack.storage.image.db;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.Event;
import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine.State;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.utils.db.GenericDaoBase;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.UpdateBuilder;
@Component
@ -45,6 +50,9 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
private SearchBuilder<VolumeDataStoreVO> storeSearch;
private SearchBuilder<VolumeDataStoreVO> cacheSearch;
private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
@Inject
DataStoreManager storeMgr;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@ -186,4 +194,45 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
sc.setParameters("destroyed", true);
return listIncludingRemovedBy(sc);
}
@Override
public void duplicateCacheRecordsOnRegionStore(long storeId) {
// find all records on image cache
List<DataStore> cacheStores = storeMgr.listImageCacheStores();
if (cacheStores == null || cacheStores.size() == 0) {
return;
}
List<VolumeDataStoreVO> vols = new ArrayList<VolumeDataStoreVO>();
for (DataStore store : cacheStores) {
// check if the volume is stored there
vols.addAll(listByStoreId(store.getId()));
}
// create an entry for each record, but with empty install path since the content is not yet on region-wide store yet
if (vols != null) {
s_logger.info("Duplicate " + vols.size() + " volume cache store records to region store");
for (VolumeDataStoreVO vol : vols) {
VolumeDataStoreVO volStore = findByStoreVolume(storeId, vol.getVolumeId());
if (volStore != null) {
s_logger.info("There is already entry for volume " + vol.getVolumeId() + " on region store " + storeId);
continue;
}
s_logger.info("Persisting an entry for volume " + vol.getVolumeId() + " on region store " + storeId);
VolumeDataStoreVO vs = new VolumeDataStoreVO();
vs.setVolumeId(vol.getVolumeId());
vs.setDataStoreId(storeId);
vs.setState(vol.getState());
vs.setDownloadPercent(vol.getDownloadPercent());
vs.setDownloadState(vol.getDownloadState());
vs.setSize(vol.getSize());
vs.setPhysicalSize(vol.getPhysicalSize());
vs.setErrorString(vol.getErrorString());
vs.setRefCnt(vol.getRefCnt());
persist(vs);
// increase ref_cnt so that this will not be recycled before the content is pushed to region-wide store
vol.incrRefCnt();
this.update(vol.getId(), vol);
}
}
}
}

33
engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeDataFactoryImpl.java
View File

@ -18,8 +18,13 @@
*/
package org.apache.cloudstack.storage.volume;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
@ -27,7 +32,6 @@ import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.springframework.stereotype.Component;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.VolumeVO;
@ -58,13 +62,13 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
if (storeRole == DataStoreRole.Image) {
VolumeDataStoreVO volumeStore = volumeStoreDao.findByVolume(volumeId);
if (volumeStore != null) {
DataStore store = this.storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image);
DataStore store = storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image);
vol = VolumeObject.getVolumeObject(store, volumeVO);
}
} else {
// Primary data store
if (volumeVO.getPoolId() != null) {
DataStore store = this.storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary);
DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary);
vol = VolumeObject.getVolumeObject(store, volumeVO);
}
}
@ -82,11 +86,11 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
DataStore store = null;
VolumeDataStoreVO volumeStore = volumeStoreDao.findByVolume(volumeId);
if (volumeStore != null) {
store = this.storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image);
store = storeMgr.getDataStore(volumeStore.getDataStoreId(), DataStoreRole.Image);
}
vol = VolumeObject.getVolumeObject(store, volumeVO);
} else {
DataStore store = this.storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary);
DataStore store = storeMgr.getDataStore(volumeVO.getPoolId(), DataStoreRole.Primary);
vol = VolumeObject.getVolumeObject(store, volumeVO);
}
return vol;
@ -99,4 +103,23 @@ public class VolumeDataFactoryImpl implements VolumeDataFactory {
return vol;
}
@Override
public List<VolumeInfo> listVolumeOnCache(long volumeId) {
List<VolumeInfo> cacheVols = new ArrayList<VolumeInfo>();
// find all image cache stores for this zone scope
List<DataStore> cacheStores = storeMgr.listImageCacheStores();
if (cacheStores == null || cacheStores.size() == 0) {
return cacheVols;
}
for (DataStore store : cacheStores) {
// check if the volume is stored there
VolumeDataStoreVO volStore = volumeStoreDao.findByStoreVolume(store.getId(), volumeId);
if (volStore != null) {
VolumeInfo vol = getVolume(volumeId, store);
cacheVols.add(vol);
}
}
return cacheVols;
}
}

36
plugins/hypervisors/simulator/src/org/apache/cloudstack/storage/datastore/lifecycle/SimulatorImageStoreLifeCycleImpl.java
View File

@ -20,12 +20,15 @@
package org.apache.cloudstack.storage.datastore.lifecycle;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
import com.cloud.utils.UriUtils;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
@ -34,13 +37,13 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.Map;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.hypervisor.Hypervisor;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
import com.cloud.utils.UriUtils;
public class SimulatorImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
private static final Logger s_logger = Logger.getLogger(SimulatorImageStoreLifeCycleImpl.class);
@ -129,4 +132,13 @@ public class SimulatorImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
public boolean deleteDataStore(DataStore store) {
return false;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

9
plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackImageStoreLifeCycleImpl.java
View File

@ -161,4 +161,13 @@ public class CloudStackImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
public boolean deleteDataStore(DataStore store) {
return false;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return imageStoreHelper.convertToStagingStore(store);
}
}

12
plugins/storage/image/s3/src/org/apache/cloudstack/storage/datastore/lifecycle/S3ImageStoreLifeCycleImpl.java
View File

@ -22,6 +22,8 @@ import java.util.Map;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
@ -32,7 +34,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle;
import org.apache.log4j.Logger;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
@ -131,4 +132,13 @@ public class S3ImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
public boolean deleteDataStore(DataStore store) {
return false;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

9
plugins/storage/image/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SampleImageStoreLifeCycleImpl.java
View File

@ -79,4 +79,13 @@ public class SampleImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
public boolean deleteDataStore(DataStore store) {
return false;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

30
plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/lifecycle/SwiftImageStoreLifeCycleImpl.java
View File

@ -16,11 +16,13 @@
// under the License.
package org.apache.cloudstack.storage.datastore.lifecycle;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
@ -30,11 +32,12 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreHelper;
import org.apache.cloudstack.storage.image.datastore.ImageStoreProviderManager;
import org.apache.cloudstack.storage.image.store.lifecycle.ImageStoreLifeCycle;
import org.apache.log4j.Logger;
import javax.inject.Inject;
import java.util.HashMap;
import java.util.Map;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.DataStoreRole;
import com.cloud.storage.ScopeType;
public class SwiftImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
@ -113,4 +116,13 @@ public class SwiftImageStoreLifeCycleImpl implements ImageStoreLifeCycle {
public boolean deleteDataStore(DataStore store) {
return false;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

17
plugins/storage/volume/default/src/org/apache/cloudstack/storage/datastore/lifecycle/CloudStackPrimaryDataStoreLifeCycleImpl.java
View File

@ -433,7 +433,7 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
List<HostVO> poolHosts = new ArrayList<HostVO>();
for (HostVO host : hosts) {
try {
this.storageMgr.connectHostToSharedPool(host.getId(), dataStore.getId());
storageMgr.connectHostToSharedPool(host.getId(), dataStore.getId());
poolHosts.add(host);
} catch (Exception e) {
s_logger.warn("Unable to establish a connection between " + host + " and " + dataStore, e);
@ -444,20 +444,20 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
primaryDataStoreDao.expunge(dataStore.getId());
throw new CloudRuntimeException("Failed to create storage pool as it is not accessible to hosts.");
}
this.dataStoreHelper.attachZone(dataStore, hypervisorType);
dataStoreHelper.attachZone(dataStore, hypervisorType);
return true;
}
@Override
public boolean maintain(DataStore dataStore) {
storagePoolAutmation.maintain(dataStore);
this.dataStoreHelper.maintain(dataStore);
dataStoreHelper.maintain(dataStore);
return true;
}
@Override
public boolean cancelMaintain(DataStore store) {
this.dataStoreHelper.cancelMaintain(store);
dataStoreHelper.cancelMaintain(store);
storagePoolAutmation.cancelMaintain(store);
return true;
}
@ -513,4 +513,13 @@ public class CloudStackPrimaryDataStoreLifeCycleImpl implements PrimaryDataStore
dataStoreHelper.attachHost(store, scope, existingInfo);
return true;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

34
plugins/storage/volume/sample/src/org/apache/cloudstack/storage/datastore/lifecycle/SamplePrimaryDataStoreLifeCycleImpl.java
View File

@ -18,12 +18,18 @@
*/
package org.apache.cloudstack.storage.datastore.lifecycle;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.StoragePoolStatus;
import org.apache.cloudstack.engine.subsystem.api.storage.*;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.command.AttachPrimaryDataStoreCmd;
import org.apache.cloudstack.storage.command.CreatePrimaryDataStoreCmd;
import org.apache.cloudstack.storage.datastore.PrimaryDataStoreProviderManager;
@ -31,9 +37,11 @@ import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
import javax.inject.Inject;
import java.util.List;
import java.util.Map;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.host.HostVO;
import com.cloud.host.dao.HostDao;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.storage.StoragePoolStatus;
public class SamplePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLifeCycle {
@Inject
@ -119,4 +127,12 @@ public class SamplePrimaryDataStoreLifeCycleImpl implements PrimaryDataStoreLife
return false;
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

254
plugins/storage/volume/solidfire/src/org/apache/cloudstack/storage/datastore/lifecycle/SolidFirePrimaryDataStoreLifeCycle.java
View File

@ -25,27 +25,28 @@ import java.util.StringTokenizer;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreParameters;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao;
import org.apache.cloudstack.storage.datastore.util.SolidFireUtil;
import org.apache.cloudstack.storage.volume.datastore.PrimaryDataStoreHelper;
import org.apache.log4j.Logger;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.dc.DataCenterVO;
import com.cloud.dc.dao.DataCenterDao;
import com.cloud.host.HostVO;
import com.cloud.hypervisor.Hypervisor.HypervisorType;
import com.cloud.agent.api.StoragePoolInfo;
import com.cloud.resource.ResourceManager;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.storage.StorageManager;
import com.cloud.storage.StoragePoolAutomation;
import com.cloud.storage.Storage.StoragePoolType;
import com.cloud.utils.exception.CloudRuntimeException;
public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeCycle {
@ -58,39 +59,39 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
@Inject StorageManager _storageMgr;
@Inject private StoragePoolAutomation storagePoolAutomation;
@Inject private StoragePoolDetailsDao storagePoolDetailsDao;
private static final int DEFAULT_MANAGEMENT_PORT = 443;
private static final int DEFAULT_STORAGE_PORT = 3260;
// invoked to add primary storage that is based on the SolidFire plug-in
@Override
public DataStore initialize(Map<String, Object> dsInfos) {
String url = (String)dsInfos.get("url");
Long zoneId = (Long)dsInfos.get("zoneId");
String url = (String)dsInfos.get("url");
Long zoneId = (Long)dsInfos.get("zoneId");
String storagePoolName = (String) dsInfos.get("name");
String providerName = (String)dsInfos.get("providerName");
Long capacityBytes = (Long)dsInfos.get("capacityBytes");
Long capacityIops = (Long)dsInfos.get("capacityIops");
String tags = (String)dsInfos.get("tags");
Map<String, String> details = (Map<String, String>)dsInfos.get("details");
String storageVip = getStorageVip(url);
int storagePort = getStoragePort(url);
DataCenterVO zone = zoneDao.findById(zoneId);
String uuid = SolidFireUtil.PROVIDER_NAME + "_" + zone.getUuid() + "_" + storageVip;
String storageVip = getStorageVip(url);
int storagePort = getStoragePort(url);
DataCenterVO zone = zoneDao.findById(zoneId);
String uuid = SolidFireUtil.PROVIDER_NAME + "_" + zone.getUuid() + "_" + storageVip;
if (capacityBytes == null || capacityBytes <= 0) {
throw new IllegalArgumentException("'capacityBytes' must be present and greater than 0.");
}
if (capacityIops == null || capacityIops <= 0) {
throw new IllegalArgumentException("'capacityIops' must be present and greater than 0.");
}
if (capacityIops == null || capacityIops <= 0) {
throw new IllegalArgumentException("'capacityIops' must be present and greater than 0.");
}
PrimaryDataStoreParameters parameters = new PrimaryDataStoreParameters();
parameters.setHost(storageVip);
parameters.setPort(storagePort);
parameters.setPath(getModifiedUrl(url));
@ -106,16 +107,16 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
parameters.setHypervisorType(HypervisorType.Any);
parameters.setTags(tags);
parameters.setDetails(details);
String managementVip = getManagementVip(url);
int managementPort = getManagementPort(url);
details.put(SolidFireUtil.MANAGEMENT_VIP, managementVip);
details.put(SolidFireUtil.MANAGEMENT_PORT, String.valueOf(managementPort));
String clusterAdminUsername = getValue(SolidFireUtil.CLUSTER_ADMIN_USERNAME, url);
String clusterAdminPassword = getValue(SolidFireUtil.CLUSTER_ADMIN_PASSWORD, url);
details.put(SolidFireUtil.CLUSTER_ADMIN_USERNAME, clusterAdminUsername);
details.put(SolidFireUtil.CLUSTER_ADMIN_PASSWORD, clusterAdminPassword);
@ -181,140 +182,140 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
// this adds a row in the cloud.storage_pool table for this SolidFire cluster
return dataStoreHelper.createPrimaryDataStore(parameters);
}
// remove the clusterAdmin and password key/value pairs
private String getModifiedUrl(String originalUrl)
{
StringBuilder sb = new StringBuilder();
String delimiter = ";";
StringTokenizer st = new StringTokenizer(originalUrl, delimiter);
while (st.hasMoreElements()) {
String token = st.nextElement().toString().toUpperCase();
if (token.startsWith(SolidFireUtil.MANAGEMENT_VIP.toUpperCase()) ||
token.startsWith(SolidFireUtil.STORAGE_VIP.toUpperCase())) {
sb.append(token).append(delimiter);
}
}
String modifiedUrl = sb.toString();
int lastIndexOf = modifiedUrl.lastIndexOf(delimiter);
if (lastIndexOf == (modifiedUrl.length() - delimiter.length())) {
return modifiedUrl.substring(0, lastIndexOf);
}
return modifiedUrl;
StringBuilder sb = new StringBuilder();
String delimiter = ";";
StringTokenizer st = new StringTokenizer(originalUrl, delimiter);
while (st.hasMoreElements()) {
String token = st.nextElement().toString().toUpperCase();
if (token.startsWith(SolidFireUtil.MANAGEMENT_VIP.toUpperCase()) ||
token.startsWith(SolidFireUtil.STORAGE_VIP.toUpperCase())) {
sb.append(token).append(delimiter);
}
}
String modifiedUrl = sb.toString();
int lastIndexOf = modifiedUrl.lastIndexOf(delimiter);
if (lastIndexOf == (modifiedUrl.length() - delimiter.length())) {
return modifiedUrl.substring(0, lastIndexOf);
}
return modifiedUrl;
}
private String getManagementVip(String url)
{
return getVip(SolidFireUtil.MANAGEMENT_VIP, url);
}
private String getStorageVip(String url)
{
return getVip(SolidFireUtil.STORAGE_VIP, url);
}
private int getManagementPort(String url)
{
return getPort(SolidFireUtil.MANAGEMENT_VIP, url, DEFAULT_MANAGEMENT_PORT);
}
private int getStoragePort(String url)
{
return getPort(SolidFireUtil.STORAGE_VIP, url, DEFAULT_STORAGE_PORT);
}
private String getVip(String keyToMatch, String url)
{
String delimiter = ":";
String storageVip = getValue(keyToMatch, url);
int index = storageVip.indexOf(delimiter);
if (index != -1)
{
return storageVip.substring(0, index);
}
return storageVip;
String delimiter = ":";
String storageVip = getValue(keyToMatch, url);
int index = storageVip.indexOf(delimiter);
if (index != -1)
{
return storageVip.substring(0, index);
}
return storageVip;
}
private int getPort(String keyToMatch, String url, int defaultPortNumber)
{
String delimiter = ":";
String storageVip = getValue(keyToMatch, url);
int index = storageVip.indexOf(delimiter);
int portNumber = defaultPortNumber;
if (index != -1) {
String port = storageVip.substring(index + delimiter.length());
try {
portNumber = Integer.parseInt(port);
}
catch (NumberFormatException ex) {
throw new IllegalArgumentException("Invalid URL format (port is not an integer)");
}
}
return portNumber;
String delimiter = ":";
String storageVip = getValue(keyToMatch, url);
int index = storageVip.indexOf(delimiter);
int portNumber = defaultPortNumber;
if (index != -1) {
String port = storageVip.substring(index + delimiter.length());
try {
portNumber = Integer.parseInt(port);
}
catch (NumberFormatException ex) {
throw new IllegalArgumentException("Invalid URL format (port is not an integer)");
}
}
return portNumber;
}
private String getValue(String keyToMatch, String url)
{
String delimiter1 = ";";
String delimiter2 = "=";
StringTokenizer st = new StringTokenizer(url, delimiter1);
while (st.hasMoreElements()) {
String token = st.nextElement().toString();
int index = token.indexOf(delimiter2);
if (index == -1)
{
throw new RuntimeException("Invalid URL format");
}
String key = token.substring(0, index);
if (key.equalsIgnoreCase(keyToMatch)) {
String valueToReturn = token.substring(index + delimiter2.length());
return valueToReturn;
}
}
throw new RuntimeException("Key not found in URL");
String delimiter1 = ";";
String delimiter2 = "=";
StringTokenizer st = new StringTokenizer(url, delimiter1);
while (st.hasMoreElements()) {
String token = st.nextElement().toString();
int index = token.indexOf(delimiter2);
if (index == -1)
{
throw new RuntimeException("Invalid URL format");
}
String key = token.substring(0, index);
if (key.equalsIgnoreCase(keyToMatch)) {
String valueToReturn = token.substring(index + delimiter2.length());
return valueToReturn;
}
}
throw new RuntimeException("Key not found in URL");
}
// do not implement this method for SolidFire's plug-in
@Override
public boolean attachHost(DataStore store, HostScope scope, StoragePoolInfo existingInfo) {
return true; // should be ignored for zone-wide-only plug-ins like SolidFire's
}
// do not implement this method for SolidFire's plug-in
@Override
public boolean attachCluster(DataStore store, ClusterScope scope) {
return true; // should be ignored for zone-wide-only plug-ins like SolidFire's
}
@Override
public boolean attachZone(DataStore dataStore, ZoneScope scope, HypervisorType hypervisorType) {
dataStoreHelper.attachZone(dataStore);
dataStoreHelper.attachZone(dataStore);
List<HostVO> xenServerHosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType.XenServer, scope.getScopeId());
List<HostVO> kvmHosts = _resourceMgr.listAllUpAndEnabledHostsInOneZoneByHypervisor(HypervisorType.KVM, scope.getScopeId());
List<HostVO> hosts = new ArrayList<HostVO>();
@ -328,11 +329,11 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
} catch (Exception e) {
s_logger.warn("Unable to establish a connection between " + host + " and " + dataStore, e);
}
}
}
return true;
}
@Override
public boolean maintain(DataStore dataStore) {
storagePoolAutomation.maintain(dataStore);
@ -340,7 +341,7 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
return true;
}
@Override
public boolean cancelMaintain(DataStore store) {
dataStoreHelper.cancelMaintain(store);
@ -348,10 +349,19 @@ public class SolidFirePrimaryDataStoreLifeCycle implements PrimaryDataStoreLifeC
return true;
}
// invoked to delete primary storage that is based on the SolidFire plug-in
@Override
public boolean deleteDataStore(DataStore store) {
return dataStoreHelper.deletePrimaryDataStore(store);
}
/* (non-Javadoc)
* @see org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle#migrateToObjectStore(org.apache.cloudstack.engine.subsystem.api.storage.DataStore)
*/
@Override
public boolean migrateToObjectStore(DataStore store) {
return false;
}
}

71
server/src/com/cloud/server/ManagementServerImpl.java
View File

@ -42,6 +42,9 @@ import javax.crypto.spec.SecretKeySpec;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.affinity.AffinityGroupProcessor;
@ -156,6 +159,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStore
import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd;
import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd;
import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.PrepareSecondaryStorageForMigrationCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd;
import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd;
@ -437,8 +441,6 @@ import org.apache.cloudstack.storage.datastore.db.ImageStoreVO;
import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.utils.identity.ManagementServerNode;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.GetVncPortAnswer;
@ -2225,21 +2227,21 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
if (domainName != null) {
String updatedDomainPath = getUpdatedDomainPath(domain.getPath(), domainName);
updateDomainChildren(domain, updatedDomainPath);
domain.setName(domainName);
domain.setPath(updatedDomainPath);
}
if (networkDomain != null) {
if (networkDomain.isEmpty()) {
domain.setNetworkDomain(null);
} else {
domain.setNetworkDomain(networkDomain);
}
}
_domainDao.update(domainId, domain);
if (domainName != null) {
String updatedDomainPath = getUpdatedDomainPath(domain.getPath(), domainName);
updateDomainChildren(domain, updatedDomainPath);
domain.setName(domainName);
domain.setPath(updatedDomainPath);
}
if (networkDomain != null) {
if (networkDomain.isEmpty()) {
domain.setNetworkDomain(null);
} else {
domain.setNetworkDomain(networkDomain);
}
}
_domainDao.update(domainId, domain);
}
});
@ -2840,6 +2842,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
cmdList.add(CreateSecondaryStagingStoreCmd.class);
cmdList.add(ListSecondaryStagingStoresCmd.class);
cmdList.add(DeleteSecondaryStagingStoreCmd.class);
cmdList.add(PrepareSecondaryStorageForMigrationCmd.class);
cmdList.add(CreateApplicationLoadBalancerCmd.class);
cmdList.add(ListApplicationLoadBalancersCmd.class);
cmdList.add(DeleteApplicationLoadBalancerCmd.class);
@ -3683,24 +3686,24 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
for (HostVO h : hosts) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Changing password for host name = " + h.getName());
}
// update password for this host
DetailVO nv = _detailsDao.findDetail(h.getId(), ApiConstants.USERNAME);
if (nv.getValue().equals(cmd.getUsername())) {
DetailVO nvp = _detailsDao.findDetail(h.getId(), ApiConstants.PASSWORD);
nvp.setValue(DBEncryptionUtil.encrypt(cmd.getPassword()));
_detailsDao.persist(nvp);
} else {
// if one host in the cluster has diff username then
// rollback to maintain consistency
throw new InvalidParameterValueException(
"The username is not same for all hosts, please modify passwords for individual hosts.");
}
for (HostVO h : hosts) {
if (s_logger.isDebugEnabled()) {
s_logger.debug("Changing password for host name = " + h.getName());
}
// update password for this host
DetailVO nv = _detailsDao.findDetail(h.getId(), ApiConstants.USERNAME);
if (nv.getValue().equals(cmd.getUsername())) {
DetailVO nvp = _detailsDao.findDetail(h.getId(), ApiConstants.PASSWORD);
nvp.setValue(DBEncryptionUtil.encrypt(cmd.getPassword()));
_detailsDao.persist(nvp);
} else {
// if one host in the cluster has diff username then
// rollback to maintain consistency
throw new InvalidParameterValueException(
"The username is not same for all hosts, please modify passwords for individual hosts.");
}
}
}
});
}
@ -3883,7 +3886,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe
@Inject
public void setStoragePoolAllocators(List<StoragePoolAllocator> storagePoolAllocators) {
this._storagePoolAllocators = storagePoolAllocators;
_storagePoolAllocators = storagePoolAllocators;
}
public LockMasterListener getLockMasterListener() {

74
server/src/com/cloud/storage/StorageManagerImpl.java
View File

@ -43,6 +43,7 @@ import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
@ -158,15 +159,14 @@ import com.cloud.utils.db.DB;
import com.cloud.utils.db.GenericSearchBuilder;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.db.JoinBuilder;
import com.cloud.utils.db.TransactionCallback;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.db.JoinBuilder.JoinType;
import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Op;
import com.cloud.utils.db.Transaction;
import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionLegacy;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine.State;
@ -1247,6 +1247,29 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
return (PrimaryDataStoreInfo) dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary);
}
@Override
@DB
public ImageStore prepareSecondaryStorageForObjectStoreMigration(Long storeId) throws ResourceUnavailableException, InsufficientCapacityException {
// Verify that image store exists
ImageStoreVO store = _imageStoreDao.findById(storeId);
if (store == null) {
throw new InvalidParameterValueException("Image store with id " + storeId + " doesn't exist");
} else if (!store.getProviderName().equals(DataStoreProvider.NFS_IMAGE)) {
throw new InvalidParameterValueException("We only support migrate NFS secondary storage to use object store!");
}
_accountMgr.checkAccessAndSpecifyAuthority(CallContext.current().getCallingAccount(), store.getDataCenterId());
DataStoreProvider provider = dataStoreProviderMgr.getDataStoreProvider(store.getProviderName());
DataStoreLifeCycle lifeCycle = provider.getDataStoreLifeCycle();
DataStore secStore = dataStoreMgr.getDataStore(storeId, DataStoreRole.Image);
lifeCycle.migrateToObjectStore(secStore);
// update store_role in template_store_ref and snapshot_store_ref to ImageCache
_templateStoreDao.updateStoreRoleToCachce(storeId);
_snapshotStoreDao.updateStoreRoleToCache(storeId);
// converted to an image cache store
return (ImageStore)_dataStoreMgr.getDataStore(storeId, DataStoreRole.ImageCache);
}
protected class StorageGarbageCollector extends ManagedContextRunnable {
public StorageGarbageCollector() {
@ -1722,9 +1745,20 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
// store
associateCrosszoneTemplatesToZone(dcId);
// duplicate cache store records to region wide storage
if (scopeType == ScopeType.REGION) {
duplicateCacheStoreRecordsToRegionStore(store.getId());
}
return (ImageStore) _dataStoreMgr.getDataStore(store.getId(), DataStoreRole.Image);
}
private void duplicateCacheStoreRecordsToRegionStore(long storeId) {
_templateStoreDao.duplicateCacheRecordsOnRegionStore(storeId);
_snapshotStoreDao.duplicateCacheRecordsOnRegionStore(storeId);
_volumeStoreDao.duplicateCacheRecordsOnRegionStore(storeId);
}
private void associateCrosszoneTemplatesToZone(Long zoneId) {
VMTemplateZoneVO tmpltZone;
@ -1785,14 +1819,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
// first delete from image_store_details table, we need to do that since
// we are not actually deleting record from main
// image_data_store table, so delete cascade will not work
_imageStoreDetailsDao.deleteDetails(storeId);
_snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.Image);
_volumeStoreDao.deletePrimaryRecordsForStore(storeId);
_templateStoreDao.deletePrimaryRecordsForStore(storeId);
_imageStoreDao.remove(storeId);
// first delete from image_store_details table, we need to do that since
// we are not actually deleting record from main
// image_data_store table, so delete cascade will not work
_imageStoreDetailsDao.deleteDetails(storeId);
_snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.Image);
_volumeStoreDao.deletePrimaryRecordsForStore(storeId);
_templateStoreDao.deletePrimaryRecordsForStore(storeId);
_imageStoreDao.remove(storeId);
}
});
@ -1897,14 +1931,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
// first delete from image_store_details table, we need to do that since
// we are not actually deleting record from main
// image_data_store table, so delete cascade will not work
_imageStoreDetailsDao.deleteDetails(storeId);
_snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.ImageCache);
_volumeStoreDao.deletePrimaryRecordsForStore(storeId);
_templateStoreDao.deletePrimaryRecordsForStore(storeId);
_imageStoreDao.remove(storeId);
// first delete from image_store_details table, we need to do that since
// we are not actually deleting record from main
// image_data_store table, so delete cascade will not work
_imageStoreDetailsDao.deleteDetails(storeId);
_snapshotStoreDao.deletePrimaryRecordsForStore(storeId, DataStoreRole.ImageCache);
_volumeStoreDao.deletePrimaryRecordsForStore(storeId);
_templateStoreDao.deletePrimaryRecordsForStore(storeId);
_imageStoreDao.remove(storeId);
}
});

125
server/src/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -411,29 +411,29 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
return Transaction.execute(new TransactionCallback<VolumeVO>() {
@Override
public VolumeVO doInTransaction(TransactionStatus status) {
VolumeVO volume = new VolumeVO(volumeName, zoneId, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK);
volume.setPoolId(null);
volume.setDataCenterId(zoneId);
volume.setPodId(null);
volume.setAccountId(owner.getAccountId());
volume.setDomainId(owner.getDomainId());
long diskOfferingId = _diskOfferingDao.findByUniqueName("Cloud.com-Custom").getId();
volume.setDiskOfferingId(diskOfferingId);
// volume.setSize(size);
volume.setInstanceId(null);
volume.setUpdated(new Date());
volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId());
volume.setFormat(ImageFormat.valueOf(format));
volume = _volsDao.persist(volume);
CallContext.current().setEventDetails("Volume Id: " + volume.getId());
// Increment resource count during allocation; if actual creation fails,
// decrement it
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
return volume;
}
VolumeVO volume = new VolumeVO(volumeName, zoneId, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK);
volume.setPoolId(null);
volume.setDataCenterId(zoneId);
volume.setPodId(null);
volume.setAccountId(owner.getAccountId());
volume.setDomainId(owner.getDomainId());
long diskOfferingId = _diskOfferingDao.findByUniqueName("Cloud.com-Custom").getId();
volume.setDiskOfferingId(diskOfferingId);
// volume.setSize(size);
volume.setInstanceId(null);
volume.setUpdated(new Date());
volume.setDomainId((owner == null) ? Domain.ROOT_DOMAIN : owner.getDomainId());
volume.setFormat(ImageFormat.valueOf(format));
volume = _volsDao.persist(volume);
CallContext.current().setEventDetails("Volume Id: " + volume.getId());
// Increment resource count during allocation; if actual creation fails,
// decrement it
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
return volume;
}
});
}
@ -615,42 +615,42 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
return Transaction.execute(new TransactionCallback<VolumeVO>() {
@Override
public VolumeVO doInTransaction(TransactionStatus status) {
VolumeVO volume = new VolumeVO(userSpecifiedName, -1, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK);
volume.setPoolId(null);
volume.setDataCenterId(zoneId);
volume.setPodId(null);
volume.setAccountId(ownerId);
volume.setDomainId(((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId()));
volume.setDiskOfferingId(diskOfferingId);
volume.setSize(size);
volume.setMinIops(minIops);
volume.setMaxIops(maxIops);
volume.setInstanceId(null);
volume.setUpdated(new Date());
volume.setDomainId((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId());
volume.setDisplayVolume(displayVolumeEnabled);
if (parentVolume != null) {
volume.setTemplateId(parentVolume.getTemplateId());
volume.setFormat(parentVolume.getFormat());
} else {
volume.setTemplateId(null);
}
volume = _volsDao.persist(volume);
if (cmd.getSnapshotId() == null) {
// for volume created from snapshot, create usage event after volume creation
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId,
null, size, Volume.class.getName(), volume.getUuid());
}
CallContext.current().setEventDetails("Volume Id: " + volume.getId());
// Increment resource count during allocation; if actual creation fails,
// decrement it
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize()));
return volume;
}
VolumeVO volume = new VolumeVO(userSpecifiedName, -1, -1, -1, -1, new Long(-1), null, null, 0, Volume.Type.DATADISK);
volume.setPoolId(null);
volume.setDataCenterId(zoneId);
volume.setPodId(null);
volume.setAccountId(ownerId);
volume.setDomainId(((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId()));
volume.setDiskOfferingId(diskOfferingId);
volume.setSize(size);
volume.setMinIops(minIops);
volume.setMaxIops(maxIops);
volume.setInstanceId(null);
volume.setUpdated(new Date());
volume.setDomainId((caller == null) ? Domain.ROOT_DOMAIN : caller.getDomainId());
volume.setDisplayVolume(displayVolumeEnabled);
if (parentVolume != null) {
volume.setTemplateId(parentVolume.getTemplateId());
volume.setFormat(parentVolume.getFormat());
} else {
volume.setTemplateId(null);
}
volume = _volsDao.persist(volume);
if (cmd.getSnapshotId() == null) {
// for volume created from snapshot, create usage event after volume creation
UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VOLUME_CREATE, volume.getAccountId(), volume.getDataCenterId(), volume.getId(), volume.getName(), diskOfferingId,
null, size, Volume.class.getName(), volume.getUuid());
}
CallContext.current().setEventDetails("Volume Id: " + volume.getId());
// Increment resource count during allocation; if actual creation fails,
// decrement it
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.primary_storage, new Long(volume.getSize()));
return volume;
}
});
}
@ -957,6 +957,13 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
AsyncCallFuture<VolumeApiResult> future2 = volService.expungeVolumeAsync(volOnSecondary);
future2.get();
}
// delete all cache entries for this volume
List<VolumeInfo> cacheVols = volFactory.listVolumeOnCache(volume.getId());
for (VolumeInfo volOnCache : cacheVols) {
s_logger.info("Delete volume from image cache store: " + volOnCache.getDataStore().getName());
volOnCache.delete();
}
} catch (Exception e) {
s_logger.warn("Failed to expunge volume:", e);
return false;

29
server/src/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -25,6 +25,8 @@ import java.util.concurrent.ExecutionException;
import javax.ejb.Local;
import javax.inject.Inject;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.command.user.iso.DeleteIsoCmd;
import org.apache.cloudstack.api.command.user.iso.RegisterIsoCmd;
import org.apache.cloudstack.api.command.user.template.DeleteTemplateCmd;
@ -44,7 +46,6 @@ import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.async.AsyncRpcContext;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.alert.AlertManager;
@ -56,11 +57,11 @@ import com.cloud.event.UsageEventUtils;
import com.cloud.exception.InvalidParameterValueException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.org.Grouping;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage.ImageFormat;
import com.cloud.storage.Storage.TemplateType;
import com.cloud.storage.TemplateProfile;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.ScopeType;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VMTemplateZoneVO;
import com.cloud.storage.dao.VMTemplateZoneDao;
@ -182,7 +183,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
// find all eligible image stores for this zone scope
List<DataStore> imageStores = this.storeMgr.getImageStoresByScope(new ZoneScope(profile.getZoneId()));
List<DataStore> imageStores = storeMgr.getImageStoresByScope(new ZoneScope(profile.getZoneId()));
if ( imageStores == null || imageStores.size() == 0 ){
throw new CloudRuntimeException("Unable to find image store to download template "+ profile.getTemplate());
}
@ -205,12 +206,12 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
}
TemplateInfo tmpl = this.imageFactory.getTemplate(template.getId(), imageStore);
TemplateInfo tmpl = imageFactory.getTemplate(template.getId(), imageStore);
CreateTemplateContext<TemplateApiResult> context = new CreateTemplateContext<TemplateApiResult>(null, tmpl);
AsyncCallbackDispatcher<HypervisorTemplateAdapter, TemplateApiResult> caller = AsyncCallbackDispatcher.create(this);
caller.setCallback(caller.getTarget().createTemplateAsyncCallBack(null, null));
caller.setContext(context);
this.imageService.createTemplateAsync(tmpl, imageStore, caller);
imageService.createTemplateAsync(tmpl, imageStore, caller);
if( !(profile.getIsPublic() || profile.getFeatured()) ){ // If private template then break
break;
}
@ -237,7 +238,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
// populated template entry
_tmpltDao.remove(template.getId());
} else {
VMTemplateVO tmplt = this._tmpltDao.findById(template.getId());
VMTemplateVO tmplt = _tmpltDao.findById(template.getId());
long accountId = tmplt.getAccountId();
if (template.getSize() != null) {
// publish usage event
@ -283,7 +284,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
VMTemplateVO template = profile.getTemplate();
// find all eligible image stores for this template
List<DataStore> imageStores = this.templateMgr.getImageStoreByTemplate(template.getId(), profile.getZoneId());
List<DataStore> imageStores = templateMgr.getImageStoreByTemplate(template.getId(), profile.getZoneId());
if (imageStores == null || imageStores.size() == 0) {
// already destroyed on image stores
s_logger.info("Unable to find image store still having template: " + template.getName()
@ -321,7 +322,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
s_logger.info("Delete template from image store: " + imageStore.getName());
AsyncCallFuture<TemplateApiResult> future = this.imageService.deleteTemplateAsync(this.imageFactory
AsyncCallFuture<TemplateApiResult> future = imageService.deleteTemplateAsync(imageFactory
.getTemplate(template.getId(), imageStore));
try {
TemplateApiResult result = future.get();
@ -350,9 +351,15 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
}
if (success) {
// delete all cache entries for this template
List<TemplateInfo> cacheTmpls = imageFactory.listTemplateOnCache(template.getId());
for (TemplateInfo tmplOnCache : cacheTmpls) {
s_logger.info("Delete template from image cache store: " + tmplOnCache.getDataStore().getName());
tmplOnCache.delete();
}
// find all eligible image stores for this template
List<DataStore> iStores = this.templateMgr.getImageStoreByTemplate(template.getId(), null);
List<DataStore> iStores = templateMgr.getImageStoreByTemplate(template.getId(), null);
if (iStores == null || iStores.size() == 0) {
// remove template from vm_templates table
if (_tmpltDao.remove(template.getId())) {
@ -380,7 +387,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
throw new InvalidParameterValueException("The DomR template cannot be deleted.");
}
if (zoneId != null && (this.storeMgr.getImageStore(zoneId) == null)) {
if (zoneId != null && (storeMgr.getImageStore(zoneId) == null)) {
throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone.");
}
@ -392,7 +399,7 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
TemplateProfile profile = super.prepareDelete(cmd);
Long zoneId = profile.getZoneId();
if (zoneId != null && (this.storeMgr.getImageStore(zoneId) == null)) {
if (zoneId != null && (storeMgr.getImageStore(zoneId) == null)) {
throw new InvalidParameterValueException("Failed to find a secondary storage in the specified zone.");
}

91
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -30,6 +30,8 @@ import javax.ejb.Local;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import org.apache.log4j.Logger;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
import org.apache.cloudstack.api.BaseListTemplateOrIsoPermissionsCmd;
import org.apache.cloudstack.api.BaseUpdateTemplateOrIsoCmd;
@ -50,7 +52,6 @@ import org.apache.cloudstack.api.command.user.template.UpdateTemplateCmd;
import org.apache.cloudstack.api.command.user.template.UpdateTemplatePermissionsCmd;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
@ -81,7 +82,6 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.log4j.Logger;
import com.cloud.agent.AgentManager;
import com.cloud.agent.api.Answer;
@ -481,9 +481,12 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
throw new InvalidParameterValueException("The " + desc + " has not been downloaded ");
}
DataObject templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
// Handle NFS to S3 object store migration case, we trigger template sync from NFS to S3 during extract template or copy template
_tmpltSvr.syncTemplateToRegionStore(templateId, tmpltStore);
return tmpltStore.createEntityExtractUrl(tmpltStoreRef.getInstallPath(), template.getFormat(), templateObject);
TemplateInfo templateObject = _tmplFactory.getTemplate(templateId, tmpltStore);
return tmpltStore.createEntityExtractUrl(templateObject.getInstallPath(), template.getFormat(), templateObject);
}
public void prepareTemplateInAllStoragePools(final VMTemplateVO template, long zoneId) {
@ -700,7 +703,15 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
throw new InvalidParameterValueException("Unable to find template with id");
}
DataStore srcSecStore = getImageStore(sourceZoneId, templateId);
if (srcSecStore == null) {
throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId);
}
if (template.isCrossZones()){
//TODO: we may need UI still enable CopyTemplate in case of cross zone template to trigger sync to region store.
// sync template from cache store to region store if it is not there, for cases where we are going to migrate existing NFS to S3.
_tmpltSvr.syncTemplateToRegionStore(templateId, srcSecStore);
s_logger.debug("Template " + templateId + " is cross-zone, don't need to copy");
return template;
}
@ -712,11 +723,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
return template;
}
DataStore srcSecStore = getImageStore(sourceZoneId, templateId);
if (srcSecStore == null) {
throw new InvalidParameterValueException("There is no template " + templateId + " in zone " + sourceZoneId);
}
_accountMgr.checkAccess(caller, AccessType.ModifyEntry, true, template);
boolean success = copy(userId, template, srcSecStore, dstZone);
@ -783,7 +789,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
try {
StoragePool pool = (StoragePool) this._dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId());
StoragePool pool = (StoragePool) _dataStoreMgr.getPrimaryDataStore(templatePoolVO.getPoolId());
VMTemplateVO template = _tmpltDao.findByIdIncludingRemoved(templatePoolVO.getTemplateId());
if (s_logger.isDebugEnabled()) {
@ -1286,27 +1292,27 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
Account owner = _accountMgr.getAccount(ownerId);
final Domain domain = _domainDao.findById(owner.getDomainId());
if ("add".equalsIgnoreCase(operation)) {
final List<String> accountNamesFinal = accountNames;
final List<String> accountNamesFinal = accountNames;
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
for (String accountName : accountNamesFinal) {
Account permittedAccount = _accountDao.findActiveAccount(accountName, domain.getId());
if (permittedAccount != null) {
if (permittedAccount.getId() == caller.getId()) {
continue; // don't grant permission to the template
// owner, they implicitly have permission
}
LaunchPermissionVO existingPermission = _launchPermissionDao.findByTemplateAndAccount(id, permittedAccount.getId());
if (existingPermission == null) {
LaunchPermissionVO launchPermission = new LaunchPermissionVO(id, permittedAccount.getId());
_launchPermissionDao.persist(launchPermission);
}
} else {
throw new InvalidParameterValueException("Unable to grant a launch permission to account " + accountName + " in domain id=" + domain.getUuid()
+ ", account not found. " + "No permissions updated, please verify the account names and retry.");
}
Account permittedAccount = _accountDao.findActiveAccount(accountName, domain.getId());
if (permittedAccount != null) {
if (permittedAccount.getId() == caller.getId()) {
continue; // don't grant permission to the template
// owner, they implicitly have permission
}
LaunchPermissionVO existingPermission = _launchPermissionDao.findByTemplateAndAccount(id, permittedAccount.getId());
if (existingPermission == null) {
LaunchPermissionVO launchPermission = new LaunchPermissionVO(id, permittedAccount.getId());
_launchPermissionDao.persist(launchPermission);
}
} else {
throw new InvalidParameterValueException("Unable to grant a launch permission to account " + accountName + " in domain id=" + domain.getUuid()
+ ", account not found. " + "No permissions updated, please verify the account names and retry.");
}
}
}
});
} else if ("remove".equalsIgnoreCase(operation)) {
@ -1436,23 +1442,23 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
Transaction.execute(new TransactionCallbackNoReturn() {
@Override
public void doInTransactionWithoutResult(TransactionStatus status) {
// template_store_ref entries should have been removed using our
// DataObject.processEvent command in case of failure, but clean
// it up here to avoid
// some leftovers which will cause removing template from
// vm_template table fail.
_tmplStoreDao.deletePrimaryRecordsForTemplate(templateId);
// Remove the template_zone_ref record
_tmpltZoneDao.deletePrimaryRecordsForTemplate(templateId);
// Remove the template record
_tmpltDao.expunge(templateId);
// decrement resource count
if (accountId != null) {
_resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template);
// template_store_ref entries should have been removed using our
// DataObject.processEvent command in case of failure, but clean
// it up here to avoid
// some leftovers which will cause removing template from
// vm_template table fail.
_tmplStoreDao.deletePrimaryRecordsForTemplate(templateId);
// Remove the template_zone_ref record
_tmpltZoneDao.deletePrimaryRecordsForTemplate(templateId);
// Remove the template record
_tmpltDao.expunge(templateId);
// decrement resource count
if (accountId != null) {
_resourceLimitMgr.decrementResourceCount(accountId, ResourceType.template);
_resourceLimitMgr.decrementResourceCount(accountId, ResourceType.secondary_storage, new Long(volumeFinal != null ? volumeFinal.getSize()
: snapshotFinal.getSize()));
}
}
}
});
@ -1714,6 +1720,7 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
return stores;
}
@Override
public VMTemplateVO updateTemplate(UpdateIsoCmd cmd) {
return updateTemplateOrIso(cmd);
@ -1843,6 +1850,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
@Inject
public void setTemplateAdapters(List<TemplateAdapter> adapters) {
this._adapters = adapters;
_adapters = adapters;
}
}

1
tools/apidoc/gen_toc.py
View File

@ -148,6 +148,7 @@ known_categories = {
'createSecondaryStagingStore': 'Image Store',
'deleteSecondaryStagingStore': 'Image Store',
'listSecondaryStagingStores': 'Image Store',
'prepareSecondaryStorageForMigration' : 'Image Store',
'InternalLoadBalancer': 'Internal LB',
'DeploymentPlanners': 'Configuration',
'PortableIp': 'Portable IP',