etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

volume upload: persisting the volume metadata 

on calling GetUploadParamsForVolume, persisting the metadata to db
    validating the account limits and incrementing the appropriate limits
    encoded the metadata on management server using preshared key
This commit is contained in:
Rajani Karuturi 2014-12-18 17:50:51 +05:30
parent 36c0c38ab8
commit 9bb6cf8452
9 changed files with 213 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/src/com/cloud/storage/VolumeApiService.java
View File

@ -22,6 +22,7 @@ import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.ExtractVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd;
@ -29,6 +30,9 @@ import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd;
import com.cloud.exception.ConcurrentOperationException;
import com.cloud.exception.ResourceAllocationException;
import com.cloud.user.Account;
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import java.net.MalformedURLException;
public interface VolumeApiService {
/**
@ -72,6 +76,8 @@ public interface VolumeApiService {
*/
Volume uploadVolume(UploadVolumeCmd cmd) throws ResourceAllocationException;
GetUploadParamsResponse uploadVolume(GetUploadParamsForVolumeCmd cmd) throws ResourceAllocationException, MalformedURLException;
boolean deleteVolume(long volumeId, Account caller) throws ConcurrentOperationException;
Volume attachVolumeToVM(AttachVolumeCmd command);

17
api/src/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java
View File

@ -19,9 +19,8 @@
package org.apache.cloudstack.api.command.user.volume;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.UUID;
import com.cloud.exception.ResourceAllocationException;
import org.apache.cloudstack.api.APICommand;
import org.apache.cloudstack.api.AbstractGetUploadParamsCmd;
import org.apache.cloudstack.api.ApiConstants;
@ -57,18 +56,14 @@ public class GetUploadParamsForVolumeCmd extends AbstractGetUploadParamsCmd {
@Override
public void execute() throws ServerApiException {
// TODO Auto-generated method stub
try {
GetUploadParamsResponse response = createGetUploadParamsResponse(
UUID.fromString("C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"),
new URL("https://1-2-3-4.xyz.com/upload/C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"),
"TKPFeuz2nHmE/kcREEu24mnj1MrLdzOeJIHXR9HLIGgk56bkRJHaD0RRL2lds1rKKhrro4/PuleEh4YhRinhxaAmPpU4e55eprG8gTCX0ItyFAtlZViVdKXMew5Dfp4Qg8W9I1/IsDJd2Kas9/ftDQLiemAlPt0uS7Ou6asOCpifnBaKvhM4UGEjHSnni1KhBzjgEyDW3Y42HKJSSv58Sgmxl9LCewBX8vtn9tXKr+j4afj7Jlh7DFhyo9HOPC5ogR4hPBKqP7xF9tHxAyq6YqfBzsng3Xwe+Pb8TU1kFHg1l2DM4tY6ooW2h8lOhWUkrJu4hOAOeTeRtCjW3H452NKoeA1M8pKWuqMo5zRMti2u2hNZs0YY2yOy8oWMMG+lG0hvIlajqEU=",
"2014-10-17T12:00:00+0530", "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9");
GetUploadParamsResponse response = _volumeService.uploadVolume(this);
response.setResponseName(getCommandName());
setResponseObject(response);
} catch (MalformedURLException e) {
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "malformedurl exception: " + e.getMessage());
} catch (MalformedURLException | ResourceAllocationException e) {
s_logger.error("exception while uploading volume", e);
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "exception while uploading a volume: " + e.getMessage());
}
}

13
core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java
View File

@ -31,6 +31,9 @@ public class TemplateOrVolumePostUploadCommand {
this.endPoint = endPoint;
}
public TemplateOrVolumePostUploadCommand() {
}
public DataObject getDataObject() {
return dataObject;
}
@ -56,7 +59,7 @@ public class TemplateOrVolumePostUploadCommand {
return false;
}
TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand) o;
TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand)o;
return dataObject.equals(that.dataObject) && endPoint.equals(that.endPoint);
@ -69,10 +72,8 @@ public class TemplateOrVolumePostUploadCommand {
return result;
}
@Override public String toString() {
return "TemplateOrVolumePostUploadCommand{" +
"dataObject=" + dataObject +
", endPoint=" + endPoint +
'}';
@Override
public String toString() {
return "TemplateOrVolumePostUploadCommand{" + "dataObject=" + dataObject + ", endPoint=" + endPoint + '}';
}
}

2
engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
View File

@ -94,6 +94,8 @@ public interface VolumeService {
AsyncCallFuture<VolumeApiResult> registerVolume(VolumeInfo volume, DataStore store);
public EndPoint registerVolumeForPostUpload(VolumeInfo volume, DataStore store);
AsyncCallFuture<VolumeApiResult> resize(VolumeInfo volume);
void resizeVolumeOnHypervisor(long volumeId, long newSize, long destHostId, String instanceName);

14
engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
View File

@ -1223,6 +1223,20 @@ public class VolumeServiceImpl implements VolumeService {
return future;
}
@Override
public EndPoint registerVolumeForPostUpload(VolumeInfo volume, DataStore store) {
DataObject volumeOnStore = store.create(volume);
volumeOnStore.processEvent(Event.CreateOnlyRequested);
EndPoint ep = _epSelector.select(store);
if (ep == null) {
s_logger.warn("There is no secondary storage VM for image store " + store.getName());
return null;
}
return ep;
}
protected Void registerVolumeCallback(AsyncCallbackDispatcher<VolumeServiceImpl, CreateCmdResult> callback, CreateVolumeContext<VolumeApiResult> context) {
CreateCmdResult result = callback.getResult();
try {

103
server/src/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -16,7 +16,10 @@
// under the License.
package com.cloud.storage;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@ -26,6 +29,15 @@ import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
import com.cloud.utils.EncryptionUtil;
import com.google.gson.ExclusionStrategy;
import com.google.gson.FieldAttributes;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd;
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
@ -146,6 +158,8 @@ import com.cloud.vm.dao.UserVmDao;
import com.cloud.vm.dao.VMInstanceDao;
import com.cloud.vm.snapshot.VMSnapshotVO;
import com.cloud.vm.snapshot.dao.VMSnapshotDao;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiService, VmWorkJobHandler {
private final static Logger s_logger = Logger.getLogger(VolumeApiServiceImpl.class);
@ -261,6 +275,70 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
return volume;
}
@Override
@DB
@ActionEvent(eventType = EventTypes.EVENT_VOLUME_UPLOAD, eventDescription = "uploading volume for post upload", async = true)
public GetUploadParamsResponse uploadVolume(GetUploadParamsForVolumeCmd cmd) throws ResourceAllocationException, MalformedURLException {
Account caller = CallContext.current().getCallingAccount();
long ownerId = cmd.getEntityOwnerId();
Account owner = _entityMgr.findById(Account.class, ownerId);
Long zoneId = cmd.getZoneId();
String volumeName = cmd.getName();
String format = cmd.getFormat();
Long diskOfferingId = cmd.getDiskOfferingId();
String imageStoreUuid = cmd.getImageStoreUuid();
DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId);
validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId);
VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, cmd.getFormat(), diskOfferingId);
VolumeInfo vol = volFactory.getVolume(volume.getId());
RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), cmd.getFormat());
vol.addPayload(payload);
EndPoint ep = volService.registerVolumeForPostUpload(vol, store);
TemplateOrVolumePostUploadCommand command = new TemplateOrVolumePostUploadCommand(vol, ep);
GetUploadParamsResponse response = new GetUploadParamsResponse();
String url = "https://" + command.getEndPoint().getPublicAddr() + "/upload/" + command.getDataObject().getUuid();
response.setPostURL(new URL(url));
response.setId(UUID.fromString(command.getDataObject().getUuid()));
/*
* TODO: hardcoding the timeout to current + 60 min for now. This needs to goto the database
*/
DateTime currentDateTime = new DateTime(DateTimeZone.UTC);
currentDateTime.plusHours(1);
String expires = currentDateTime.toString();
response.setTimeout(expires);
String key = _configDao.getValue(Config.SSVMPSK.key());
/*
* encoded metadata using the post upload config ssh key
*/
final List<String> fieldExclusions = Arrays.asList("s_logger");
Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy() {
@Override public boolean shouldSkipField(FieldAttributes f) {
return f.getDeclaringClass() == Logger.class;
}
@Override public boolean shouldSkipClass(Class<?> clazz) {
return false;
}
}).create();
String jsonPayload = gson.toJson(command);
response.setMetadata(EncryptionUtil.encodeData(jsonPayload, key));
/*
* signature calculated on the url, expiry, metadata.
*/
response.setSignature(EncryptionUtil.generateSignature(jsonPayload + url + expires, key));
return response;
}
private boolean validateVolume(Account caller, long ownerId, Long zoneId, String volumeName, String url,
String format, Long diskOfferingId) throws ResourceAllocationException {
@ -282,8 +360,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + zoneId);
}
if (url.toLowerCase().contains("file://")) {
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
//validating the url only when url is not null. url can be null incase of form based post upload
if (url != null ) {
if( url.toLowerCase().contains("file://")) {
throw new InvalidParameterValueException("File:// type urls are currently unsupported");
}
UriUtils.validateUrl(format, url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
} else {
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage);
}
ImageFormat imgfmt = ImageFormat.valueOf(format.toUpperCase());
@ -291,12 +377,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
throw new IllegalArgumentException("Image format is incorrect " + format + ". Supported formats are " + EnumUtils.listValues(ImageFormat.values()));
}
UriUtils.validateUrl(format, url);
// Check that the resource limit for secondary storage won't be exceeded
_resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
// Check that the the disk offering specified is valid
if (diskOfferingId != null) {
DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId);
@ -357,7 +437,12 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
// Increment resource count during allocation; if actual creation fails,
// decrement it
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume);
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
//url can be null incase of postupload
if(url!=null) {
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url));
} else {
_resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage);
}
return volume;
}

39
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -19,8 +19,6 @@ package com.cloud.template;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
@ -31,17 +29,18 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import javax.ejb.Local;
import javax.inject.Inject;
import javax.naming.ConfigurationException;
import com.cloud.utils.EncryptionUtil;
import com.google.gson.ExclusionStrategy;
import com.google.gson.FieldAttributes;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
@ -363,17 +362,25 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
String expires = currentDateTime.toString();
response.setTimeout(expires);
String key = _configDao.getValue(Config.SSVMPSK.key());
/*
* encoded metadata using the post upload config ssh key
*/
Gson gson = new Gson();
Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy() {
@Override public boolean shouldSkipField(FieldAttributes f) {
return f.getDeclaredType().getClass().isInstance(Logger.class);
}
@Override public boolean shouldSkipClass(Class<?> clazz) {
return false;
}
}).create();
String jsonPayload = gson.toJson(payload);
response.setMetadata(encodeData(jsonPayload));
response.setMetadata(EncryptionUtil.encodeData(jsonPayload, key));
/*
* signature calculated on the url, expiry, metadata.
*/
response.setSignature(encodeData(jsonPayload+url+expires));
response.setSignature(EncryptionUtil.generateSignature(jsonPayload + url + expires, key));
return response;
} else {
@ -381,22 +388,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager,
}
}
private String encodeData(String data) {
String key = _configDao.getValue(Config.SSVMPSK.key());
try {
final Mac mac = Mac.getInstance("HmacSHA1");
final SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA1");
mac.init(keySpec);
mac.update(data.getBytes());
final byte[] encryptedBytes = mac.doFinal();
final String computedSignature = Base64.encodeBase64String(encryptedBytes);
return computedSignature;
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
s_logger.error("exception occured which encoding the data.", e);
return null;
}
}
@Override
public DataStore getImageStore(String storeUuid, Long zoneId) {

13
utils/pom.xml
View File

@ -183,19 +183,6 @@
</excludes>
</configuration>
</plugin>
<plugin>
<groupId>com.mycila</groupId>
<artifactId>license-maven-plugin</artifactId>
<executions>
<execution>
<id>cloudstack-checklicence</id>
<phase>process-classes</phase>
<goals>
<goal>check</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
<resources>
<resource>

69
utils/src/com/cloud/utils/EncryptionUtil.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.utils;
import org.apache.commons.codec.binary.Base64;
import org.apache.log4j.Logger;
import org.jasypt.encryption.pbe.PBEStringEncryptor;
import org.jasypt.encryption.pbe.StandardPBEStringEncryptor;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
public class EncryptionUtil {
public static final Logger s_logger = Logger.getLogger(EncryptionUtil.class.getName());
private static PBEStringEncryptor encryptor;
private static void initialize(String key) {
StandardPBEStringEncryptor standardPBEStringEncryptor = new StandardPBEStringEncryptor();
standardPBEStringEncryptor.setAlgorithm("PBEWITHSHA1ANDDESEDE");
standardPBEStringEncryptor.setPassword(key);
encryptor = standardPBEStringEncryptor;
}
public static String encodeData(String data, String key) {
if (encryptor == null) {
initialize(key);
}
return encryptor.encrypt(data);
}
public static String decodeData(String encodedData, String key) {
if (encryptor == null) {
initialize(key);
}
return encryptor.decrypt(encodedData);
}
public static String generateSignature(String data, String key) {
try {
final Mac mac = Mac.getInstance("HmacSHA1");
final SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA1");
mac.init(keySpec);
mac.update(data.getBytes());
final byte[] encryptedBytes = mac.doFinal();
return Base64.encodeBase64String(encryptedBytes);
} catch (NoSuchAlgorithmException | InvalidKeyException e) {
s_logger.error("exception occurred which encoding the data.", e);
return null;
}
}
}