etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'volume-upload' of https://git-wip-us.apache.org/repos/asf/cloudstack into volume-upload

This commit is contained in:
sailajamada 2015-03-23 09:59:58 +05:30
parent f22bd7a79e 018023c1ef
commit 48078289f5
12 changed files with 170 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/src/org/apache/cloudstack/api/command/user/template/GetUploadParamsForTemplateCmd.java
View File

@ -109,6 +109,9 @@ public class GetUploadParamsForTemplateCmd extends AbstractGetUploadParamsCmd {
}
public Boolean isDynamicallyScalable() {
if (isDynamicallyScalable == null) {
return Boolean.FALSE;
}
return isDynamicallyScalable;
}

4
core/src/com/cloud/storage/template/HttpTemplateDownloader.java
View File

@ -28,6 +28,7 @@ import java.net.URISyntaxException;
import java.net.URL;
import java.util.Date;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.commons.httpclient.Credentials;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
@ -45,7 +46,6 @@ import org.apache.log4j.Logger;
import org.apache.cloudstack.managed.context.ManagedContextRunnable;
import org.apache.cloudstack.storage.command.DownloadCommand.ResourceType;
import org.apache.cloudstack.utils.template.TemplateUtils;
import com.cloud.agent.api.storage.Proxy;
import com.cloud.storage.StorageLayer;
@ -259,7 +259,7 @@ public class HttpTemplateDownloader extends ManagedContextRunnable implements Te
} catch (URISyntaxException e) {
s_logger.warn("Invalid download url: " + getDownloadUrl() + ", This should not happen since we have validated the url before!!");
}
String unsupportedFormat = TemplateUtils.checkTemplateFormat(file.getAbsolutePath(), uripath);
String unsupportedFormat = ImageStoreUtil.checkTemplateFormat(file.getAbsolutePath(), uripath);
if (unsupportedFormat == null || !unsupportedFormat.isEmpty()) {
try {
request.abort();

10
core/src/org/apache/cloudstack/storage/command/TemplateOrVolumePostUploadCommand.java
View File

@ -47,6 +47,8 @@ public class TemplateOrVolumePostUploadCommand {
String maxUploadSize;
String description;
public TemplateOrVolumePostUploadCommand(long entityId, String entityUUID, String absolutePath, String checksum, String type, String name, String imageFormat, String dataTo,
String dataToRole) {
this.entityId = entityId;
@ -166,4 +168,12 @@ public class TemplateOrVolumePostUploadCommand {
public void setMaxUploadSize(String maxUploadSize) {
this.maxUploadSize = maxUploadSize;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

4
server/src/com/cloud/storage/VolumeApiServiceImpl.java
View File

@ -29,7 +29,6 @@ import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
import com.cloud.utils.EncryptionUtil;
import com.cloud.utils.ImageStoreUtil;
import com.cloud.utils.db.TransactionCallbackWithException;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@ -39,6 +38,7 @@ import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.engine.subsystem.api.storage.DataObject;
import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.log4j.Logger;
import org.apache.cloudstack.api.command.user.volume.AttachVolumeCmd;
import org.apache.cloudstack.api.command.user.volume.CreateVolumeCmd;
@ -333,7 +333,7 @@ public class VolumeApiServiceImpl extends ManagerBase implements VolumeApiServic
* encoded metadata using the post upload config key
*/
TemplateOrVolumePostUploadCommand command =
new TemplateOrVolumePostUploadCommand(vol.getId(), vol.getUuid(), volumeStore.getInstallPath(), volumeStore.getChecksum(), vol.getType().toString(),
new TemplateOrVolumePostUploadCommand(vol.getId(), vol.getUuid(), volumeStore.getInstallPath(), cmd.getChecksum(), vol.getType().toString(),
vol.getName(), vol.getFormat().toString(), dataObject.getDataStore().getUri(),
dataObject.getDataStore().getRole().toString());
command.setLocalPath(volumeStore.getLocalDownloadPath());

3
server/src/com/cloud/template/HypervisorTemplateAdapter.java
View File

@ -258,12 +258,13 @@ public class HypervisorTemplateAdapter extends TemplateAdapterBase {
}
TemplateOrVolumePostUploadCommand payload = new TemplateOrVolumePostUploadCommand(template.getId(), template.getUuid(), tmpl.getInstallPath(), tmpl.getChecksum(), tmpl
.getType().toString(), template.getName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), templateOnStore.getDataStore().getRole()
.getType().toString(), template.getUniqueName(), template.getFormat().toString(), templateOnStore.getDataStore().getUri(), templateOnStore.getDataStore().getRole()
.toString());
//using the existing max template size configuration
payload.setMaxUploadSize(_configDao.getValue(Config.MaxTemplateAndIsoSize.key()));
payload.setRemoteEndPoint(ep.getPublicAddr());
payload.setRequiresHvm(template.requiresHvm());
payload.setDescription(template.getDisplayText());
payloads.add(payload);
}
_resourceLimitMgr.incrementResourceCount(profile.getAccountId(), ResourceType.template);

2
server/src/com/cloud/template/TemplateManagerImpl.java
View File

@ -35,13 +35,13 @@ import javax.naming.ConfigurationException;
import com.cloud.storage.ImageStoreUploadMonitorImpl;
import com.cloud.utils.EncryptionUtil;
import com.cloud.utils.ImageStoreUtil;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.cloudstack.api.command.user.template.GetUploadParamsForTemplateCmd;
import org.apache.cloudstack.api.response.GetUploadParamsResponse;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;

49
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/HttpUploadServerHandler.java
View File

@ -28,6 +28,8 @@ import java.util.Map;
import java.util.Map.Entry;
import org.apache.cloudstack.storage.template.UploadEntity;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import com.cloud.exception.InvalidParameterValueException;
@ -75,6 +77,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
private String uuid;
private boolean fileReceived = false;
private static final String HEADER_SIGNATURE = "X-signature";
private static final String HEADER_METADATA = "X-metadata";
@ -92,6 +96,16 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
if (decoder != null) {
decoder.cleanFiles();
}
fileReceived = false;
}
@Override
public void channelInactive(ChannelHandlerContext ctx) throws Exception {
if (!fileReceived) {
String message = "file receive failed or connection closed prematurely.";
logger.error(message);
storageResource.updateStateMapWithError(uuid, message);
}
}
@Override
@ -191,15 +205,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
return;
}
if (chunk instanceof LastHttpContent) {
try {
readFileUploadData();
writeResponse(ctx.channel(), HttpResponseStatus.OK);
reset();
} catch (InvalidParameterValueException e) {
logger.error("error during the file install.", e);
responseContent.append("\n").append(e.getMessage());
writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
writeResponse(ctx.channel(), readFileUploadData());
reset();
}
}
}
@ -213,7 +220,7 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
decoder = null;
}
private void readFileUploadData() throws IOException {
private HttpResponseStatus readFileUploadData() throws IOException {
while (decoder.hasNext()) {
InterfaceHttpData data = decoder.next();
if (data != null) {
@ -222,8 +229,24 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
if (data.getHttpDataType() == HttpDataType.FileUpload) {
FileUpload fileUpload = (FileUpload) data;
if (fileUpload.isCompleted()) {
responseContent.append("upload successful.");
storageResource.postUpload(uuid, fileUpload.getFile().getName());
fileReceived = true;
String format = ImageStoreUtil.checkTemplateFormat(fileUpload.getFile().getAbsolutePath(), fileUpload.getFilename());
if(StringUtils.isNotBlank(format)) {
String errorString = "File type mismatch between the sent file and the actual content. Received: " + format;
logger.error(errorString);
responseContent.append(errorString);
storageResource.updateStateMapWithError(uuid, errorString);
return HttpResponseStatus.BAD_REQUEST;
}
String status = storageResource.postUpload(uuid, fileUpload.getFile().getName());
if (status != null) {
responseContent.append(status);
storageResource.updateStateMapWithError(uuid, status);
return HttpResponseStatus.INTERNAL_SERVER_ERROR;
} else {
responseContent.append("upload successful.");
return HttpResponseStatus.OK;
}
}
}
} finally {
@ -231,6 +254,8 @@ public class HttpUploadServerHandler extends SimpleChannelInboundHandler<HttpObj
}
}
}
responseContent.append("received entity is not a file");
return HttpResponseStatus.UNPROCESSABLE_ENTITY;
}
private void writeResponse(Channel channel, HttpResponseStatus statusCode) {

38
services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java
View File

@ -66,8 +66,10 @@ import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
import org.apache.cloudstack.storage.template.UploadEntity;
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
@ -2634,6 +2636,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
uploadEntity.setHvm(cmd.getRequiresHvm());
uploadEntity.setChksum(cmd.getChecksum());
uploadEntity.setMaxSizeInGB(maxSizeInGB);
uploadEntity.setDescription(cmd.getDescription());
// create a install dir
if (!_storage.exists(installPathPrefix)) {
_storage.mkdir(installPathPrefix);
@ -2661,28 +2664,41 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
UploadEntity.ResourceType resourceType = uploadEntity.getResourceType();
String fileSavedTempLocation = uploadEntity.getInstallPathPrefix() + "/" + filename;
//String checkSum = computeCheckSum(originalTemplate);
//if (checkSum == null) {
// s_logger.warn("Something wrong happened when try to calculate the checksum of downloaded template!");
//}
//dnld.setCheckSum(checkSum);
String uploadedFileExtension = FilenameUtils.getExtension(filename);
String userSelectedFormat= "vhd";
if(uploadedFileExtension.equals("zip") || uploadedFileExtension.equals("bz2") || uploadedFileExtension.equals("gz")) {
userSelectedFormat += "." + uploadedFileExtension;
}
String formatError = ImageStoreUtil.checkTemplateFormat(fileSavedTempLocation, userSelectedFormat);
if(StringUtils.isNotBlank(formatError)) {
String errorString = "File type mismatch between uploaded file and selected format. Selected file format: " + uploadEntity.getFormat() + ". Received: " + formatError;
s_logger.error(errorString);
return errorString;
}
int imgSizeGigs = getSizeInGB(_storage.getSize(fileSavedTempLocation));
int maxSize = uploadEntity.getMaxSizeInGB();
if(imgSizeGigs > maxSize) {
throw new InvalidParameterValueException("Maximum file upload size exceeded. Physical file size: "+imgSizeGigs+"GB. Maximum allowed size: "+maxSize+"GB.");
String errorMessage = "Maximum file upload size exceeded. Physical file size: " + imgSizeGigs + "GB. Maximum allowed size: " + maxSize + "GB.";
s_logger.error(errorMessage);
return errorMessage;
}
imgSizeGigs++; // add one just in case
long timeout = (long)imgSizeGigs * installTimeoutPerGig;
Script scr = new Script(getScriptLocation(resourceType), timeout, s_logger);
scr.add("-s", Integer.toString(imgSizeGigs));
scr.add("-S", Long.toString(UploadEntity.s_maxTemplateSize));
//if (uploadEntity.getDescription() != null && dnld.getDescription().length() > 1) {
// scr.add("-d", dnld.getDescription());
//}
if (uploadEntity.getDescription() != null && uploadEntity.getDescription().length() > 1) {
scr.add("-d", uploadEntity.getDescription());
}
if (uploadEntity.isHvm()) {
scr.add("-h");
}
String checkSum = uploadEntity.getChksum();
if (StringUtils.isNotBlank(checkSum)) {
scr.add("-c", checkSum);
}
// add options common to ISO and template
String extension = uploadEntity.getFormat().getFileExtension();
@ -2733,7 +2749,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
} catch (IOException e) {
s_logger.warn("Something is wrong with template location " + resourcePath, e);
loc.purge();
return "Unable to download due to " + e.getMessage();
return "Unable to upload due to " + e.getMessage();
}
Map<String, Processor> processors = _dlMgr.getProcessors();
@ -2773,7 +2789,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
return _ssvmPSK;
}
private void updateStateMapWithError(String uuid,String errorMessage) {
public void updateStateMapWithError(String uuid,String errorMessage) {
UploadEntity uploadEntity=null;
if (uploadEntityStateMap.get(uuid)!=null) {
uploadEntity=uploadEntityStateMap.get(uuid);

9
services/secondary-storage/server/src/org/apache/cloudstack/storage/template/UploadEntity.java
View File

@ -32,6 +32,7 @@ public class UploadEntity {
private String chksum;
private long physicalSize;
private int maxSizeInGB;
private String description;
public static enum ResourceType {
VOLUME, TEMPLATE
@ -180,4 +181,12 @@ public class UploadEntity {
public void setMaxSizeInGB(int maxSizeInGB) {
this.maxSizeInGB = maxSizeInGB;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
}

39
utils/src/com/cloud/utils/ImageStoreUtil.java
View File

@ -1,39 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.cloud.utils;
import org.apache.log4j.Logger;
public class ImageStoreUtil {
public static final Logger s_logger = Logger.getLogger(ImageStoreUtil.class.getName());
public static String generatePostUploadUrl(String ssvmUrlDomain, String ipAddress, String uuid) {
String hostname = ipAddress;
//if ssvm url domain is present, use it to construct hostname in the format 1-2-3-4.domain
// if the domain name is not present, ssl validation fails and has to be ignored
if(StringUtils.isNotBlank(ssvmUrlDomain)) {
hostname = ipAddress.replace(".", "-");
hostname = hostname + ssvmUrlDomain.substring(1);
}
//only https works with postupload and url format is fixed
return "https://" + hostname + "/upload/" + uuid;
}
}

65
utils/src/org/apache/cloudstack/utils/template/TemplateUtils.java → utils/src/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java
View File

@ -1,30 +1,43 @@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//
package org.apache.cloudstack.utils.template;
import org.apache.log4j.Logger;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.cloudstack.utils.imagestore;
import com.cloud.utils.script.Script;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
public class TemplateUtils {
public static final Logger s_logger = Logger.getLogger(TemplateUtils.class.getName());
public class ImageStoreUtil {
public static final Logger s_logger = Logger.getLogger(ImageStoreUtil.class.getName());
public static String generatePostUploadUrl(String ssvmUrlDomain, String ipAddress, String uuid) {
String hostname = ipAddress;
//if ssvm url domain is present, use it to construct hostname in the format 1-2-3-4.domain
// if the domain name is not present, ssl validation fails and has to be ignored
if(StringUtils.isNotBlank(ssvmUrlDomain)) {
hostname = ipAddress.replace(".", "-");
hostname = hostname + ssvmUrlDomain.substring(1);
}
//only https works with postupload and url format is fixed
return "https://" + hostname + "/upload/" + uuid;
}
// given a path, returns empty if path is supported image, and the file type if unsupported
// this is meant to catch things like accidental upload of ASCII text .vmdk descriptor
@ -75,7 +88,7 @@ public class TemplateUtils {
return output;
}
public static boolean isCorrectExtension(String path, String ext) {
private static boolean isCorrectExtension(String path, String ext) {
if (path.toLowerCase().endsWith(ext)
|| path.toLowerCase().endsWith(ext + ".gz")
|| path.toLowerCase().endsWith(ext + ".bz2")
@ -85,7 +98,7 @@ public class TemplateUtils {
return false;
}
public static boolean isCompressedExtension(String path) {
private static boolean isCompressedExtension(String path) {
if (path.toLowerCase().endsWith(".gz")
|| path.toLowerCase().endsWith(".bz2")
|| path.toLowerCase().endsWith(".zip")) {

38
utils/test/org/apache/cloudstack/utils/imagestore/ImageStoreUtilTest.java Normal file
View File

@ -0,0 +1,38 @@
package org.apache.cloudstack.utils.imagestore;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.UUID;
import org.junit.Assert;
import org.junit.Test;
public class ImageStoreUtilTest {
@Test
public void testgeneratePostUploadUrl() throws MalformedURLException {
String ssvmdomain = "*.realhostip.com";
String ipAddress = "10.147.28.14";
String uuid = UUID.randomUUID().toString();
//ssvm domain is not set
String url = ImageStoreUtil.generatePostUploadUrl(null, ipAddress, uuid);
assertPostUploadUrl(url, ipAddress, uuid);
//ssvm domain is set to empty value
url = ImageStoreUtil.generatePostUploadUrl("", ipAddress, uuid);
assertPostUploadUrl(url, ipAddress, uuid);
//ssvm domain is set to a valid value
url = ImageStoreUtil.generatePostUploadUrl(ssvmdomain, ipAddress, uuid);
assertPostUploadUrl(url, ipAddress.replace(".", "-") + ssvmdomain.substring(1), uuid);
}
private void assertPostUploadUrl(String urlStr, String domain, String uuid) throws MalformedURLException {
URL url = new URL(urlStr);
Assert.assertNotNull(url);
Assert.assertEquals(url.getHost(), domain);
Assert.assertEquals(url.getPath(), "/upload/" + uuid);
}
}