diff --git a/api/src/com/cloud/storage/VolumeApiService.java b/api/src/com/cloud/storage/VolumeApiService.java index 5487a4bf9b2..7ed15165b1a 100644 --- a/api/src/com/cloud/storage/VolumeApiService.java +++ b/api/src/com/cloud/storage/VolumeApiService.java @@ -22,6 +22,7 @@ import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.DetachVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ExtractVolumeCmd; +import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd; import org.apache.cloudstack.api.command.user.volume.MigrateVolumeCmd; import org.apache.cloudstack.api.command.user.volume.ResizeVolumeCmd; import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd; @@ -29,6 +30,9 @@ import org.apache.cloudstack.api.command.user.volume.UploadVolumeCmd; import com.cloud.exception.ConcurrentOperationException; import com.cloud.exception.ResourceAllocationException; import com.cloud.user.Account; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; + +import java.net.MalformedURLException; public interface VolumeApiService { /** @@ -72,6 +76,8 @@ public interface VolumeApiService { */ Volume uploadVolume(UploadVolumeCmd cmd) throws ResourceAllocationException; + GetUploadParamsResponse uploadVolume(GetUploadParamsForVolumeCmd cmd) throws ResourceAllocationException, MalformedURLException; + boolean deleteVolume(long volumeId, Account caller) throws ConcurrentOperationException; Volume attachVolumeToVM(AttachVolumeCmd command); diff --git a/api/src/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java b/api/src/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java index d6f5c7d8a61..1342ffc4748 100644 --- a/api/src/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java +++ b/api/src/org/apache/cloudstack/api/command/user/volume/GetUploadParamsForVolumeCmd.java @@ -19,9 +19,8 @@ package org.apache.cloudstack.api.command.user.volume; import java.net.MalformedURLException; -import java.net.URL; -import java.util.UUID; +import com.cloud.exception.ResourceAllocationException; import org.apache.cloudstack.api.APICommand; import org.apache.cloudstack.api.AbstractGetUploadParamsCmd; import org.apache.cloudstack.api.ApiConstants; @@ -57,18 +56,14 @@ public class GetUploadParamsForVolumeCmd extends AbstractGetUploadParamsCmd { @Override public void execute() throws ServerApiException { - // TODO Auto-generated method stub + try { - GetUploadParamsResponse response = createGetUploadParamsResponse( - UUID.fromString("C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"), - new URL("https://1-2-3-4.xyz.com/upload/C7D351D2-F167-4CC8-A9FF-3BECB0A625C4"), - "TKPFeuz2nHmE/kcREEu24mnj1MrLdzOeJIHXR9HLIGgk56bkRJHaD0RRL2lds1rKKhrro4/PuleEh4YhRinhxaAmPpU4e55eprG8gTCX0ItyFAtlZViVdKXMew5Dfp4Qg8W9I1/IsDJd2Kas9/ftDQLiemAlPt0uS7Ou6asOCpifnBaKvhM4UGEjHSnni1KhBzjgEyDW3Y42HKJSSv58Sgmxl9LCewBX8vtn9tXKr+j4afj7Jlh7DFhyo9HOPC5ogR4hPBKqP7xF9tHxAyq6YqfBzsng3Xwe+Pb8TU1kFHg1l2DM4tY6ooW2h8lOhWUkrJu4hOAOeTeRtCjW3H452NKoeA1M8pKWuqMo5zRMti2u2hNZs0YY2yOy8oWMMG+lG0hvIlajqEU=", - "2014-10-17T12:00:00+0530", "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9"); + GetUploadParamsResponse response = _volumeService.uploadVolume(this); response.setResponseName(getCommandName()); setResponseObject(response); - - } catch (MalformedURLException e) { - throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "malformedurl exception: " + e.getMessage()); + } catch (MalformedURLException | ResourceAllocationException e) { + s_logger.error("exception while uploading volume", e); + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "exception while uploading a volume: " + e.getMessage()); } } diff --git a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java index de943c4680c..d2a9577100b 100644 --- a/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java +++ b/core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java @@ -31,6 +31,9 @@ public class TemplateOrVolumePostUploadCommand { this.endPoint = endPoint; } + public TemplateOrVolumePostUploadCommand() { + } + public DataObject getDataObject() { return dataObject; } @@ -56,7 +59,7 @@ public class TemplateOrVolumePostUploadCommand { return false; } - TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand) o; + TemplateOrVolumePostUploadCommand that = (TemplateOrVolumePostUploadCommand)o; return dataObject.equals(that.dataObject) && endPoint.equals(that.endPoint); @@ -69,10 +72,8 @@ public class TemplateOrVolumePostUploadCommand { return result; } - @Override public String toString() { - return "TemplateOrVolumePostUploadCommand{" + - "dataObject=" + dataObject + - ", endPoint=" + endPoint + - '}'; + @Override + public String toString() { + return "TemplateOrVolumePostUploadCommand{" + "dataObject=" + dataObject + ", endPoint=" + endPoint + '}'; } } diff --git a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java index 171e9df0dc0..d9a75cc0da5 100644 --- a/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java +++ b/engine/api/src/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java @@ -94,6 +94,8 @@ public interface VolumeService { AsyncCallFuture registerVolume(VolumeInfo volume, DataStore store); + public EndPoint registerVolumeForPostUpload(VolumeInfo volume, DataStore store); + AsyncCallFuture resize(VolumeInfo volume); void resizeVolumeOnHypervisor(long volumeId, long newSize, long destHostId, String instanceName); diff --git a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index edb50b2833f..5d23ffe0cd1 100644 --- a/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -1223,6 +1223,20 @@ public class VolumeServiceImpl implements VolumeService { return future; } + @Override + public EndPoint registerVolumeForPostUpload(VolumeInfo volume, DataStore store) { + DataObject volumeOnStore = store.create(volume); + + volumeOnStore.processEvent(Event.CreateOnlyRequested); + + EndPoint ep = _epSelector.select(store); + if (ep == null) { + s_logger.warn("There is no secondary storage VM for image store " + store.getName()); + return null; + } + return ep; + } + protected Void registerVolumeCallback(AsyncCallbackDispatcher callback, CreateVolumeContext context) { CreateCmdResult result = callback.getResult(); try { diff --git a/server/src/com/cloud/storage/VolumeApiServiceImpl.java b/server/src/com/cloud/storage/VolumeApiServiceImpl.java index a6c6ca7b02b..5f15e3a4587 100644 --- a/server/src/com/cloud/storage/VolumeApiServiceImpl.java +++ b/server/src/com/cloud/storage/VolumeApiServiceImpl.java @@ -16,7 +16,10 @@ // under the License. package com.cloud.storage; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -26,6 +29,15 @@ import java.util.concurrent.ExecutionException; import javax.inject.Inject; +import com.cloud.utils.EncryptionUtil; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.cloudstack.api.command.user.volume.GetUploadParamsForVolumeCmd; +import org.apache.cloudstack.api.response.GetUploadParamsResponse; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.log4j.Logger; import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd; @@ -146,6 +158,8 @@ import com.cloud.vm.dao.UserVmDao; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.snapshot.VMSnapshotVO; import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiService, VmWorkJobHandler { private final static Logger s_logger = Logger.getLogger(VolumeApiServiceImpl.class); @@ -261,6 +275,70 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic return volume; } + @Override + @DB + @ActionEvent(eventType = EventTypes.EVENT_VOLUME_UPLOAD, eventDescription = "uploading volume for post upload", async = true) + public GetUploadParamsResponse uploadVolume(GetUploadParamsForVolumeCmd cmd) throws ResourceAllocationException, MalformedURLException { + Account caller = CallContext.current().getCallingAccount(); + long ownerId = cmd.getEntityOwnerId(); + Account owner = _entityMgr.findById(Account.class, ownerId); + Long zoneId = cmd.getZoneId(); + String volumeName = cmd.getName(); + String format = cmd.getFormat(); + Long diskOfferingId = cmd.getDiskOfferingId(); + String imageStoreUuid = cmd.getImageStoreUuid(); + DataStore store = _tmpltMgr.getImageStore(imageStoreUuid, zoneId); + + validateVolume(caller, ownerId, zoneId, volumeName, null, format, diskOfferingId); + + VolumeVO volume = persistVolume(owner, zoneId, volumeName, null, cmd.getFormat(), diskOfferingId); + + VolumeInfo vol = volFactory.getVolume(volume.getId()); + + RegisterVolumePayload payload = new RegisterVolumePayload(null, cmd.getChecksum(), cmd.getFormat()); + vol.addPayload(payload); + + EndPoint ep = volService.registerVolumeForPostUpload(vol, store); + + TemplateOrVolumePostUploadCommand command = new TemplateOrVolumePostUploadCommand(vol, ep); + + GetUploadParamsResponse response = new GetUploadParamsResponse(); + String url = "https://" + command.getEndPoint().getPublicAddr() + "/upload/" + command.getDataObject().getUuid(); + response.setPostURL(new URL(url)); + + response.setId(UUID.fromString(command.getDataObject().getUuid())); + + /* + * TODO: hardcoding the timeout to current + 60 min for now. This needs to goto the database + */ + DateTime currentDateTime = new DateTime(DateTimeZone.UTC); + currentDateTime.plusHours(1); + String expires = currentDateTime.toString(); + response.setTimeout(expires); + + String key = _configDao.getValue(Config.SSVMPSK.key()); + /* + * encoded metadata using the post upload config ssh key + */ + final List fieldExclusions = Arrays.asList("s_logger"); + Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy() { + @Override public boolean shouldSkipField(FieldAttributes f) { + return f.getDeclaringClass() == Logger.class; + } + @Override public boolean shouldSkipClass(Class clazz) { + return false; + } + }).create(); + String jsonPayload = gson.toJson(command); + response.setMetadata(EncryptionUtil.encodeData(jsonPayload, key)); + + /* + * signature calculated on the url, expiry, metadata. + */ + response.setSignature(EncryptionUtil.generateSignature(jsonPayload + url + expires, key)); + return response; + } + private boolean validateVolume(Account caller, long ownerId, Long zoneId, String volumeName, String url, String format, Long diskOfferingId) throws ResourceAllocationException { @@ -282,8 +360,16 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new PermissionDeniedException("Cannot perform this operation, Zone is currently disabled: " + zoneId); } - if (url.toLowerCase().contains("file://")) { - throw new InvalidParameterValueException("File:// type urls are currently unsupported"); + //validating the url only when url is not null. url can be null incase of form based post upload + if (url != null ) { + if( url.toLowerCase().contains("file://")) { + throw new InvalidParameterValueException("File:// type urls are currently unsupported"); + } + UriUtils.validateUrl(format, url); + // Check that the resource limit for secondary storage won't be exceeded + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + } else { + _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage); } ImageFormat imgfmt = ImageFormat.valueOf(format.toUpperCase()); @@ -291,12 +377,6 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic throw new IllegalArgumentException("Image format is incorrect " + format + ". Supported formats are " + EnumUtils.listValues(ImageFormat.values())); } - UriUtils.validateUrl(format, url); - - - // Check that the resource limit for secondary storage won't be exceeded - _resourceLimitMgr.checkResourceLimit(_accountMgr.getAccount(ownerId), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); - // Check that the the disk offering specified is valid if (diskOfferingId != null) { DiskOfferingVO diskOffering = _diskOfferingDao.findById(diskOfferingId); @@ -357,7 +437,12 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic // Increment resource count during allocation; if actual creation fails, // decrement it _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.volume); - _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + //url can be null incase of postupload + if(url!=null) { + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage, UriUtils.getRemoteSize(url)); + } else { + _resourceLimitMgr.incrementResourceCount(volume.getAccountId(), ResourceType.secondary_storage); + } return volume; } diff --git a/server/src/com/cloud/template/TemplateManagerImpl.java b/server/src/com/cloud/template/TemplateManagerImpl.java index 21859867afe..53fb00d8f77 100755 --- a/server/src/com/cloud/template/TemplateManagerImpl.java +++ b/server/src/com/cloud/template/TemplateManagerImpl.java @@ -19,8 +19,6 @@ package com.cloud.template; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.security.InvalidKeyException; -import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -31,17 +29,18 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; import javax.ejb.Local; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.EncryptionUtil; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd; import org.apache.cloudstack.api.response.GetUploadParamsResponse; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; @@ -363,17 +362,25 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, String expires = currentDateTime.toString(); response.setTimeout(expires); + String key = _configDao.getValue(Config.SSVMPSK.key()); /* * encoded metadata using the post upload config ssh key */ - Gson gson = new Gson(); + Gson gson = new GsonBuilder().setExclusionStrategies(new ExclusionStrategy() { + @Override public boolean shouldSkipField(FieldAttributes f) { + return f.getDeclaredType().getClass().isInstance(Logger.class); + } + @Override public boolean shouldSkipClass(Class clazz) { + return false; + } + }).create(); String jsonPayload = gson.toJson(payload); - response.setMetadata(encodeData(jsonPayload)); + response.setMetadata(EncryptionUtil.encodeData(jsonPayload, key)); /* * signature calculated on the url, expiry, metadata. */ - response.setSignature(encodeData(jsonPayload+url+expires)); + response.setSignature(EncryptionUtil.generateSignature(jsonPayload + url + expires, key)); return response; } else { @@ -381,22 +388,6 @@ public class TemplateManagerImpl extends ManagerBase implements TemplateManager, } } - private String encodeData(String data) { - String key = _configDao.getValue(Config.SSVMPSK.key()); - - try { - final Mac mac = Mac.getInstance("HmacSHA1"); - final SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA1"); - mac.init(keySpec); - mac.update(data.getBytes()); - final byte[] encryptedBytes = mac.doFinal(); - final String computedSignature = Base64.encodeBase64String(encryptedBytes); - return computedSignature; - } catch (NoSuchAlgorithmException | InvalidKeyException e) { - s_logger.error("exception occured which encoding the data.", e); - return null; - } - } @Override public DataStore getImageStore(String storeUuid, Long zoneId) { diff --git a/utils/pom.xml b/utils/pom.xml index 0bf940c0b7b..2f1fa404d26 100755 --- a/utils/pom.xml +++ b/utils/pom.xml @@ -183,19 +183,6 @@ - - com.mycila - license-maven-plugin - - - cloudstack-checklicence - process-classes - - check - - - - diff --git a/utils/src/com/cloud/utils/EncryptionUtil.java b/utils/src/com/cloud/utils/EncryptionUtil.java new file mode 100644 index 00000000000..28d781fcadb --- /dev/null +++ b/utils/src/com/cloud/utils/EncryptionUtil.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.utils; + +import org.apache.commons.codec.binary.Base64; +import org.apache.log4j.Logger; +import org.jasypt.encryption.pbe.PBEStringEncryptor; +import org.jasypt.encryption.pbe.StandardPBEStringEncryptor; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +public class EncryptionUtil { + public static final Logger s_logger = Logger.getLogger(EncryptionUtil.class.getName()); + private static PBEStringEncryptor encryptor; + + private static void initialize(String key) { + StandardPBEStringEncryptor standardPBEStringEncryptor = new StandardPBEStringEncryptor(); + standardPBEStringEncryptor.setAlgorithm("PBEWITHSHA1ANDDESEDE"); + standardPBEStringEncryptor.setPassword(key); + encryptor = standardPBEStringEncryptor; + } + + public static String encodeData(String data, String key) { + if (encryptor == null) { + initialize(key); + } + return encryptor.encrypt(data); + } + + public static String decodeData(String encodedData, String key) { + if (encryptor == null) { + initialize(key); + } + return encryptor.decrypt(encodedData); + } + + public static String generateSignature(String data, String key) { + try { + final Mac mac = Mac.getInstance("HmacSHA1"); + final SecretKeySpec keySpec = new SecretKeySpec(key.getBytes(), "HmacSHA1"); + mac.init(keySpec); + mac.update(data.getBytes()); + final byte[] encryptedBytes = mac.doFinal(); + return Base64.encodeBase64String(encryptedBytes); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + s_logger.error("exception occurred which encoding the data.", e); + return null; + } + } +}