etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CLOUDSTACK-3659 

Implement the download url expiration functionality for volumes
Signed off by : nitin mehta<nitin.mehta@citrix.com>
This commit is contained in:
Nitin Mehta 2013-07-28 17:35:10 +05:30
parent f83187585e
commit faeddc15ff
9 changed files with 127 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
engine/api/src/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java
View File

@ -40,4 +40,6 @@ StateDao<ObjectInDataStoreStateMachine.State, ObjectInDataStoreStateMachine.Even
VolumeDataStoreVO findByStoreVolume(long storeId, long volumeId, boolean lock);
List<VolumeDataStoreVO> listDestroyed(long storeId);
List<VolumeDataStoreVO> listVolumeDownloadUrls();
}

2
engine/api/src/org/apache/cloudstack/storage/image/datastore/ImageStoreEntity.java
View File

@ -43,4 +43,6 @@ public interface ImageStoreEntity extends DataStore, ImageStore {
String getMountPoint(); // get the mount point on ssvm.
String createEntityExtractUrl(String installPath, ImageFormat format, DataObject dataObject); // get the entity download URL
void deleteExtractUrl(String installPath, String url, ImageFormat format);
}

4
engine/storage/image/src/org/apache/cloudstack/storage/image/store/ImageStoreImpl.java
View File

@ -194,5 +194,9 @@ public class ImageStoreImpl implements ImageStoreEntity {
return driver.createEntityExtractUrl(this, installPath, format, dataObject);
}
@Override
public void deleteExtractUrl(String installPath, String url, ImageFormat format) {
driver.deleteEntityExtractUrl(this, installPath, url, format);
}
}

6
engine/storage/src/org/apache/cloudstack/storage/image/BaseImageStoreDriverImpl.java
View File

@ -24,6 +24,7 @@ import com.cloud.agent.api.storage.Proxy;
import com.cloud.agent.api.to.DataObjectType;
import com.cloud.agent.api.to.DataTO;
import com.cloud.configuration.dao.ConfigurationDao;
import com.cloud.storage.Storage;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.VolumeVO;
@ -246,4 +247,9 @@ public abstract class BaseImageStoreDriverImpl implements ImageStoreDriver {
@Override
public void resize(DataObject data, AsyncCompletionCallback<CreateCmdResult> callback) {
}
@Override
public void deleteEntityExtractUrl(DataStore store, String installPath, String url, Storage.ImageFormat format){
}
}

4
engine/storage/src/org/apache/cloudstack/storage/image/ImageStoreDriver.java
View File

@ -25,5 +25,9 @@ import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreDriver;
import com.cloud.storage.Storage.ImageFormat;
public interface ImageStoreDriver extends DataStoreDriver {
String createEntityExtractUrl(DataStore store, String installPath, ImageFormat format, DataObject dataObject);
void deleteEntityExtractUrl(DataStore store, String installPath, String url, ImageFormat format);
}

14
engine/storage/src/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java
View File

@ -45,6 +45,7 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
private SearchBuilder<VolumeDataStoreVO> storeSearch;
private SearchBuilder<VolumeDataStoreVO> cacheSearch;
private SearchBuilder<VolumeDataStoreVO> storeVolumeSearch;
private SearchBuilder<VolumeDataStoreVO> downloadVolumeSearch;
@Override
public boolean configure(String name, Map<String, Object> params) throws ConfigurationException {
@ -77,6 +78,12 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
updateStateSearch.and("state", updateStateSearch.entity().getState(), Op.EQ);
updateStateSearch.and("updatedCount", updateStateSearch.entity().getUpdatedCount(), Op.EQ);
updateStateSearch.done();
downloadVolumeSearch = createSearchBuilder();
downloadVolumeSearch.and("download_url", downloadVolumeSearch.entity().getExtractUrl(), Op.NNULL);
downloadVolumeSearch.and("destroyed", downloadVolumeSearch.entity().getDestroyed(), SearchCriteria.Op.EQ);
downloadVolumeSearch.done();
return true;
}
@ -186,4 +193,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase<VolumeDataStoreVO, Lo
sc.setParameters("destroyed", true);
return listIncludingRemovedBy(sc);
}
@Override
public List<VolumeDataStoreVO> listVolumeDownloadUrls() {
SearchCriteria<VolumeDataStoreVO> sc = downloadVolumeSearch.create();
sc.setParameters("destroyed", false);
return listBy(sc);
}
}

21
plugins/storage/image/default/src/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java
View File

@ -23,6 +23,10 @@ import java.util.UUID;
import javax.inject.Inject;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.storage.Storage;
import com.cloud.storage.Upload;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
@ -76,6 +80,23 @@ public class CloudStackImageStoreDriverImpl extends BaseImageStoreDriverImpl {
return generateCopyUrl(ep.getPublicAddr(), uuid);
}
@Override
public void deleteEntityExtractUrl(DataStore store, String installPath, String downloadUrl, ImageFormat format) {
// find an endpoint to send command
EndPoint ep = _epSelector.select(store);
// Create Symlink at ssvm
//CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity) store).getMountPoint(), installPath, uuid);
DeleteEntityDownloadURLCommand cmd = new DeleteEntityDownloadURLCommand(installPath, Upload.Type.VOLUME, downloadUrl, ((ImageStoreEntity) store).getMountPoint());
Answer ans = ep.sendMessage(cmd);
if (ans == null || !ans.getResult()) {
String errorString = "Unable to delete the url " + downloadUrl + " for path " + installPath + " on ssvm, " + ans.getDetails();
s_logger.error(errorString);
throw new CloudRuntimeException(errorString);
}
}
private String generateCopyUrl(String ipAddress, String uuid){
String hostname = ipAddress;

82
server/src/com/cloud/storage/StorageManagerImpl.java
View File

@ -41,6 +41,8 @@ import javax.ejb.Local;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.agent.api.storage.DeleteEntityDownloadURLCommand;
import com.cloud.utils.DateUtil;
import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd;
import org.apache.cloudstack.api.command.admin.storage.CreateSecondaryStagingStoreCmd;
@ -49,26 +51,8 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingSto
import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd;
import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd;
import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStore;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreLifeCycle;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector;
import org.apache.cloudstack.engine.subsystem.api.storage.HostScope;
import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener;
import org.apache.cloudstack.engine.subsystem.api.storage.ImageStoreProvider;
import org.apache.cloudstack.engine.subsystem.api.storage.PrimaryDataStoreInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.StoragePoolAllocator;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.TemplateService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.engine.subsystem.api.storage.*;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService.VolumeApiResult;
import org.apache.cloudstack.engine.subsystem.api.storage.ZoneScope;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao;
@ -81,6 +65,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao;
import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -257,6 +242,7 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
DataStoreManager _dataStoreMgr;
@Inject
DataStoreProviderManager _dataStoreProviderMgr;
@Inject
private TemplateService _imageSrv;
@Inject
@ -291,6 +277,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
boolean _storageCleanupEnabled;
boolean _templateCleanupEnabled = true;
int _storageCleanupInterval;
int _downloadUrlCleanupInterval;
int _downloadUrlExpirationInterval;
private int _createVolumeFromSnapshotWait;
private int _copyvolumewait;
int _storagePoolAcquisitionWaitSeconds = 1800; // 30 minutes
@ -510,6 +498,12 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
int wrks = NumbersUtil.parseInt(workers, 10);
_executor = Executors.newScheduledThreadPool(wrks, new NamedThreadFactory("StorageManager-Scavenger"));
String cleanupInterval = configs.get("extract.url.cleanup.interval");
_downloadUrlCleanupInterval = NumbersUtil.parseInt(cleanupInterval, 7200);
String urlExpirationInterval = configs.get("extract.url.expiration.interval");
_downloadUrlExpirationInterval = NumbersUtil.parseInt(urlExpirationInterval, 14400);
_agentMgr.registerForHostEvents(ComponentContext.inject(LocalStoragePoolListener.class), true, false, false);
String maxVolumeSizeInGbString = _configDao.getValue("storage.max.volume.size");
@ -571,6 +565,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
} else {
s_logger.debug("Storage cleanup is not enabled, so the storage cleanup thread is not being scheduled.");
}
_executor.scheduleWithFixedDelay(new DownloadURLGarbageCollector(), _downloadUrlCleanupInterval, _downloadUrlCleanupInterval, TimeUnit.SECONDS);
return true;
}
@ -1097,6 +1093,35 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
public void cleanupDownloadUrls(){
// find volumesOnImageStoreList with download url
List<VolumeDataStoreVO> volumesOnImageStoreList = _volumeStoreDao.listVolumeDownloadUrls();
for(VolumeDataStoreVO volumeOnImageStore : volumesOnImageStoreList){
try {
long downloadUrlCurrentAgeInSecs = DateUtil.getTimeDifference(DateUtil.now(), volumeOnImageStore.getUpdated());
if(downloadUrlCurrentAgeInSecs < _downloadUrlExpirationInterval){ // URL hasnt expired yet
continue;
}
s_logger.debug("Removing download url " + volumeOnImageStore.getExtractUrl() + " for volume id " + volumeOnImageStore.getVolumeId());
// Remove it from image store
ImageStoreEntity secStore = (ImageStoreEntity) _dataStoreMgr.getDataStore(volumeOnImageStore.getDataStoreId(), DataStoreRole.Image);
secStore.deleteExtractUrl(volumeOnImageStore.getInstallPath(), volumeOnImageStore.getExtractUrl(), ImageFormat.VHD);
// Now remove it from DB.
volumeOnImageStore.setExtractUrl(null);
_volumeStoreDao.update(volumeOnImageStore.getId(), volumeOnImageStore);
}catch(Throwable th){
s_logger.warn("caught exception while deleting download url " +volumeOnImageStore.getExtractUrl(), th);
}
}
}
@Override
@DB
public void cleanupSecondaryStorage(boolean recurring) {
@ -1262,6 +1287,25 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C
}
}
protected class DownloadURLGarbageCollector implements Runnable {
public DownloadURLGarbageCollector() {
}
@Override
public void run() {
try {
s_logger.trace("Download URL Garbage Collection Thread is running.");
cleanupDownloadUrls();
} catch (Exception e) {
s_logger.error("Caught the following Exception", e);
}
}
}
@Override
public void onManagementNodeJoined(List<ManagementServerHostVO> nodeList, long selfNodeId) {
// TODO Auto-generated method stub

11
utils/src/com/cloud/utils/DateUtil.java
View File

@ -227,6 +227,17 @@ public class DateUtil {
}
return scheduleTime.getTime();
}
public static long getTimeDifference(Date date1, Date date2){
Calendar dateCalendar1 = Calendar.getInstance();
dateCalendar1.setTime(date1);
Calendar dateCalendar2 = Calendar.getInstance();
dateCalendar2.setTime(date2);
return (dateCalendar1.getTimeInMillis() - dateCalendar2.getTimeInMillis() )/1000;
}
// test only