mirror of https://github.com/apache/cloudstack.git
Merge pull request #1331 from syed/swift-restart-fix
Fix Sync of template.properties in SwiftWhen using Swift as a secondary storage, we create a template.properties file which stores the metadata about the template. Currently the data which is present in the file is incorrect which leads to templates becoming unavailable after they are downloaded. This fix makes sure that the template.properties has the correct "path" set so that templates are available. I've also done a bit of cleanup and made the code bit more clean. * pr/1331: Fix Sync of template.properties in Swift Signed-off-by: Will Stevens <williamstevens@gmail.com>
This commit is contained in:
Will Stevens
commit
06e52e4f34
|
|
@ -156,6 +156,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager {
|
|||
// template.properties
|
||||
// there
|
||||
}
|
||||
|
||||
ts.setInstallPath(installPath);
|
||||
ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
|
||||
ts = templateDataStoreDao.persist(ts);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,178 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
package com.cloud.test;
|
||||
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.Objects;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.apache.log4j.AppenderSkeleton;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.apache.log4j.spi.LoggingEvent;
|
||||
import org.springframework.util.Assert;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static com.google.common.base.Preconditions.checkArgument;
|
||||
import static com.google.common.base.Preconditions.checkState;
|
||||
import static com.google.common.base.Strings.isNullOrEmpty;
|
||||
import static java.lang.String.format;
|
||||
import static org.apache.log4j.Level.ALL;
|
||||
import static org.apache.log4j.Level.DEBUG;
|
||||
import static org.apache.log4j.Level.ERROR;
|
||||
import static org.apache.log4j.Level.FATAL;
|
||||
import static org.apache.log4j.Level.INFO;
|
||||
import static org.apache.log4j.Level.OFF;
|
||||
|
||||
/**
|
||||
*
|
||||
* Tracks one or more patterns to determine whether or not they have been
|
||||
* logged. It uses a streaming approach to determine whether or not a message
|
||||
* has a occurred to prevent unnecessary memory consumption. Instances of this
|
||||
* of this class are created using the {@link TestAppenderBuilder}.
|
||||
*
|
||||
* To use this class, register a one or more expected patterns by level as part
|
||||
* of the test setup and retain an reference to the appender instance. After the
|
||||
* expected logging events have occurred in the test case, call
|
||||
* {@link TestAppender#assertMessagesLogged()} which will fail the test if any of the
|
||||
* expected patterns were not logged.
|
||||
*
|
||||
*/
|
||||
public final class TestAppender extends AppenderSkeleton {
|
||||
private final static String APPENDER_NAME = "test_appender";
|
||||
private final ImmutableMap<Level, Set<PatternResult>> expectedPatternResults;
|
||||
private TestAppender(final Map<Level, Set<PatternResult>> expectedPatterns) {
|
||||
super();
|
||||
expectedPatternResults = ImmutableMap.copyOf(expectedPatterns);
|
||||
}
|
||||
protected void append(LoggingEvent loggingEvent) {
|
||||
checkArgument(loggingEvent != null, "append requires a non-null loggingEvent");
|
||||
final Level level = loggingEvent.getLevel();
|
||||
checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported by append");
|
||||
for (final PatternResult patternResult : expectedPatternResults.get(level)) {
|
||||
if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches()) {
|
||||
patternResult.markFound();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void close() {
|
||||
// Do nothing ...
|
||||
}
|
||||
public boolean requiresLayout() {
|
||||
return false;
|
||||
}
|
||||
public void assertMessagesLogged() {
|
||||
final List<String> unloggedPatterns = new ArrayList<>();
|
||||
for (final Map.Entry<Level, Set<PatternResult>> expectedPatternResult : expectedPatternResults.entrySet()) {
|
||||
for (final PatternResult patternResults : expectedPatternResult.getValue()) {
|
||||
if (!patternResults.isFound()) {
|
||||
unloggedPatterns.add(format("%1$s was not logged for level %2$s",
|
||||
patternResults.getPattern().toString(), expectedPatternResult.getKey()));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!unloggedPatterns.isEmpty()) {
|
||||
//Raise an assert
|
||||
Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns));
|
||||
}
|
||||
}
|
||||
|
||||
private static final class PatternResult {
|
||||
private final Pattern pattern;
|
||||
private boolean foundFlag = false;
|
||||
private PatternResult(Pattern pattern) {
|
||||
super();
|
||||
this.pattern = pattern;
|
||||
}
|
||||
public Pattern getPattern() {
|
||||
return pattern;
|
||||
}
|
||||
public void markFound() {
|
||||
// This operation is thread-safe because the value will only ever be switched from false to true. Therefore,
|
||||
// multiple threads mutating the value for a pattern will not corrupt the value ...
|
||||
foundFlag = true;
|
||||
}
|
||||
public boolean isFound() {
|
||||
return foundFlag;
|
||||
}
|
||||
@Override
|
||||
public boolean equals(Object thatObject) {
|
||||
if (this == thatObject) {
|
||||
return true;
|
||||
}
|
||||
if (thatObject == null || getClass() != thatObject.getClass()) {
|
||||
return false;
|
||||
}
|
||||
PatternResult thatPatternResult = (PatternResult) thatObject;
|
||||
return foundFlag == thatPatternResult.foundFlag &&
|
||||
Objects.equal(pattern, thatPatternResult.pattern);
|
||||
}
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hashCode(pattern, foundFlag);
|
||||
}
|
||||
@Override
|
||||
public String toString() {
|
||||
return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(), foundFlag);
|
||||
}
|
||||
}
|
||||
|
||||
public static final class TestAppenderBuilder {
|
||||
private final Map<Level, Set<PatternResult>> expectedPatterns;
|
||||
public TestAppenderBuilder() {
|
||||
super();
|
||||
expectedPatterns = new HashMap<>();
|
||||
expectedPatterns.put(ALL, new HashSet<PatternResult>());
|
||||
expectedPatterns.put(DEBUG, new HashSet<PatternResult>());
|
||||
expectedPatterns.put(ERROR, new HashSet<PatternResult>());
|
||||
expectedPatterns.put(FATAL, new HashSet<PatternResult>());
|
||||
expectedPatterns.put(INFO, new HashSet<PatternResult>());
|
||||
expectedPatterns.put(OFF, new HashSet<PatternResult>());
|
||||
}
|
||||
public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) {
|
||||
checkArgument(level != null, "addExpectedPattern requires a non-null level");
|
||||
checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank pattern");
|
||||
checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported by " + getClass().getName());
|
||||
expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern)));
|
||||
return this;
|
||||
}
|
||||
public TestAppender build() {
|
||||
return new TestAppender(expectedPatterns);
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only
|
||||
* test appender attached to the logger.
|
||||
*
|
||||
* @param logger The logger which will be monitored by the test
|
||||
* @param testAppender The test appender to attach to {@code logger}
|
||||
*/
|
||||
public static void safeAddAppender(Logger logger, TestAppender testAppender) {
|
||||
logger.removeAppender(APPENDER_NAME);
|
||||
logger.addAppender(testAppender);
|
||||
}
|
||||
}
|
||||
|
|
@ -26,9 +26,6 @@
|
|||
<version>4.7.2-SNAPSHOT</version>
|
||||
<relativePath>../pom.xml</relativePath>
|
||||
</parent>
|
||||
<properties>
|
||||
<skipTests>true</skipTests>
|
||||
</properties>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>log4j</groupId>
|
||||
|
|
|
|||
|
|
@ -16,91 +16,7 @@
|
|||
// under the License.
|
||||
package org.apache.cloudstack.storage.resource;
|
||||
|
||||
import static com.cloud.utils.storage.S3.S3Utils.putFile;
|
||||
import static com.cloud.utils.StringUtils.join;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.apache.commons.lang.StringUtils.substringAfterLast;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.math.BigInteger;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.UnknownHostException;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
|
||||
import com.cloud.exception.InvalidParameterValueException;
|
||||
import com.cloud.storage.Storage;
|
||||
import com.cloud.storage.template.TemplateConstants;
|
||||
import com.cloud.utils.EncryptionUtil;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.channel.Channel;
|
||||
import io.netty.channel.ChannelInitializer;
|
||||
import io.netty.channel.ChannelPipeline;
|
||||
import io.netty.channel.EventLoopGroup;
|
||||
import io.netty.channel.nio.NioEventLoopGroup;
|
||||
import io.netty.channel.socket.SocketChannel;
|
||||
import io.netty.channel.socket.nio.NioServerSocketChannel;
|
||||
import io.netty.handler.codec.http.HttpContentCompressor;
|
||||
import io.netty.handler.codec.http.HttpRequestDecoder;
|
||||
import io.netty.handler.codec.http.HttpResponseEncoder;
|
||||
import io.netty.handler.logging.LogLevel;
|
||||
import io.netty.handler.logging.LoggingHandler;
|
||||
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
|
||||
import org.apache.cloudstack.storage.template.UploadEntity;
|
||||
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.utils.URLEncodedUtils;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
|
||||
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
|
||||
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
|
||||
import org.apache.cloudstack.storage.command.CopyCommand;
|
||||
import org.apache.cloudstack.storage.command.DeleteCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
|
||||
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
|
||||
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
|
||||
import org.apache.cloudstack.storage.command.UploadStatusCommand;
|
||||
import org.apache.cloudstack.storage.template.DownloadManager;
|
||||
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
|
||||
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
|
||||
import org.apache.cloudstack.storage.template.UploadManager;
|
||||
import org.apache.cloudstack.storage.template.UploadManagerImpl;
|
||||
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
|
||||
import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
||||
import org.apache.cloudstack.storage.to.VolumeObjectTO;
|
||||
|
||||
import com.cloud.agent.api.Answer;
|
||||
import com.cloud.agent.api.CheckHealthAnswer;
|
||||
import com.cloud.agent.api.CheckHealthCommand;
|
||||
|
|
@ -135,11 +51,13 @@ import com.cloud.agent.api.to.NfsTO;
|
|||
import com.cloud.agent.api.to.S3TO;
|
||||
import com.cloud.agent.api.to.SwiftTO;
|
||||
import com.cloud.exception.InternalErrorException;
|
||||
import com.cloud.exception.InvalidParameterValueException;
|
||||
import com.cloud.host.Host;
|
||||
import com.cloud.host.Host.Type;
|
||||
import com.cloud.hypervisor.Hypervisor.HypervisorType;
|
||||
import com.cloud.resource.ServerResourceBase;
|
||||
import com.cloud.storage.DataStoreRole;
|
||||
import com.cloud.storage.Storage;
|
||||
import com.cloud.storage.Storage.ImageFormat;
|
||||
import com.cloud.storage.StorageLayer;
|
||||
import com.cloud.storage.VMTemplateStorageResourceAssoc;
|
||||
|
|
@ -149,24 +67,102 @@ import com.cloud.storage.template.Processor.FormatInfo;
|
|||
import com.cloud.storage.template.QCOW2Processor;
|
||||
import com.cloud.storage.template.RawImageProcessor;
|
||||
import com.cloud.storage.template.TARProcessor;
|
||||
import com.cloud.storage.template.TemplateConstants;
|
||||
import com.cloud.storage.template.TemplateLocation;
|
||||
import com.cloud.storage.template.TemplateProp;
|
||||
import com.cloud.storage.template.VhdProcessor;
|
||||
import com.cloud.storage.template.VmdkProcessor;
|
||||
import com.cloud.utils.EncryptionUtil;
|
||||
import com.cloud.utils.NumbersUtil;
|
||||
import com.cloud.utils.storage.S3.S3Utils;
|
||||
import com.cloud.utils.SwiftUtil;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import com.cloud.utils.net.NetUtils;
|
||||
import com.cloud.utils.script.OutputInterpreter;
|
||||
import com.cloud.utils.script.Script;
|
||||
import com.cloud.utils.storage.S3.S3Utils;
|
||||
import com.cloud.vm.SecondaryStorageVm;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.GsonBuilder;
|
||||
import io.netty.bootstrap.ServerBootstrap;
|
||||
import io.netty.channel.Channel;
|
||||
import io.netty.channel.ChannelInitializer;
|
||||
import io.netty.channel.ChannelPipeline;
|
||||
import io.netty.channel.EventLoopGroup;
|
||||
import io.netty.channel.nio.NioEventLoopGroup;
|
||||
import io.netty.channel.socket.SocketChannel;
|
||||
import io.netty.channel.socket.nio.NioServerSocketChannel;
|
||||
import io.netty.handler.codec.http.HttpContentCompressor;
|
||||
import io.netty.handler.codec.http.HttpRequestDecoder;
|
||||
import io.netty.handler.codec.http.HttpResponseEncoder;
|
||||
import io.netty.handler.logging.LogLevel;
|
||||
import io.netty.handler.logging.LoggingHandler;
|
||||
import org.apache.cloudstack.framework.security.keystore.KeystoreManager;
|
||||
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
|
||||
import org.apache.cloudstack.storage.command.CopyCommand;
|
||||
import org.apache.cloudstack.storage.command.DeleteCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadProgressCommand;
|
||||
import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand;
|
||||
import org.apache.cloudstack.storage.command.UploadStatusAnswer;
|
||||
import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus;
|
||||
import org.apache.cloudstack.storage.command.UploadStatusCommand;
|
||||
import org.apache.cloudstack.storage.template.DownloadManager;
|
||||
import org.apache.cloudstack.storage.template.DownloadManagerImpl;
|
||||
import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser;
|
||||
import org.apache.cloudstack.storage.template.UploadEntity;
|
||||
import org.apache.cloudstack.storage.template.UploadManager;
|
||||
import org.apache.cloudstack.storage.template.UploadManagerImpl;
|
||||
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
|
||||
import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
||||
import org.apache.cloudstack.storage.to.VolumeObjectTO;
|
||||
import org.apache.cloudstack.utils.imagestore.ImageStoreUtil;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.io.FilenameUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.NameValuePair;
|
||||
import org.apache.http.client.HttpClient;
|
||||
import org.apache.http.client.methods.HttpGet;
|
||||
import org.apache.http.client.utils.URLEncodedUtils;
|
||||
import org.apache.http.impl.client.DefaultHttpClient;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.joda.time.DateTime;
|
||||
import org.joda.time.format.ISODateTimeFormat;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
import java.io.BufferedReader;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.FileReader;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.math.BigInteger;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.UnknownHostException;
|
||||
import java.security.MessageDigest;
|
||||
import java.security.NoSuchAlgorithmException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import static com.cloud.utils.StringUtils.join;
|
||||
import static com.cloud.utils.storage.S3.S3Utils.putFile;
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Arrays.asList;
|
||||
import static org.apache.commons.lang.StringUtils.substringAfterLast;
|
||||
|
||||
public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource {
|
||||
|
||||
private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
|
||||
public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class);
|
||||
|
||||
private static final String TEMPLATE_ROOT_DIR = "template/tmpl";
|
||||
private static final String VOLUME_ROOT_DIR = "volumes";
|
||||
|
|
@ -499,10 +495,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName;
|
||||
s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template " + destFileFullPath);
|
||||
Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath);
|
||||
String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties";
|
||||
String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp;
|
||||
File metaFile = new File(metaFileName);
|
||||
try {
|
||||
_storage.create(destFile.getAbsolutePath(), "template.properties");
|
||||
_storage.create(destFile.getAbsolutePath(), _tmpltpp);
|
||||
try ( // generate template.properties file
|
||||
FileWriter writer = new FileWriter(metaFile);
|
||||
BufferedWriter bufferWriter = new BufferedWriter(writer);
|
||||
|
|
@ -597,32 +593,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
return answer;
|
||||
}
|
||||
s_logger.debug("starting copy template to swift");
|
||||
DataTO newTemplate = answer.getNewData();
|
||||
File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl());
|
||||
SwiftTO swift = (SwiftTO)destDataStore;
|
||||
String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
|
||||
String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName());
|
||||
//upload template.properties
|
||||
File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
|
||||
if (properties.exists()) {
|
||||
SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
|
||||
}
|
||||
TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData();
|
||||
newTemplate.setDataStore(srcDataStore);
|
||||
CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
|
||||
Answer result = copyFromNfsToSwift(newCpyCmd);
|
||||
|
||||
//clean up template data on staging area
|
||||
try {
|
||||
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
|
||||
execute(deleteCommand);
|
||||
} catch (Exception e) {
|
||||
s_logger.debug("Failed to clean up staging area:", e);
|
||||
}
|
||||
cleanupStagingNfs(newTemplate);
|
||||
return result;
|
||||
|
||||
TemplateObjectTO template = new TemplateObjectTO();
|
||||
template.setPath(swiftPath);
|
||||
template.setSize(templateFile.length());
|
||||
template.setPhysicalSize(template.getSize());
|
||||
SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
|
||||
template.setFormat(snapshot.getVolume().getFormat());
|
||||
return new CopyCmdAnswer(template);
|
||||
} else if (destDataStore instanceof S3TO) {
|
||||
//create template on the same data store
|
||||
CopyCmdAnswer answer =
|
||||
|
|
@ -635,18 +613,27 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
newTemplate.setDataStore(srcDataStore);
|
||||
CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
|
||||
Answer result = copyFromNfsToS3(newCpyCmd);
|
||||
//clean up template data on staging area
|
||||
try {
|
||||
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
|
||||
execute(deleteCommand);
|
||||
} catch (Exception e) {
|
||||
s_logger.debug("Failed to clean up staging area:", e);
|
||||
}
|
||||
|
||||
cleanupStagingNfs(newTemplate);
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
s_logger.debug("Failed to create templat from snapshot");
|
||||
return new CopyCmdAnswer("Unsupported prototcol");
|
||||
s_logger.debug("Failed to create template from snapshot");
|
||||
return new CopyCmdAnswer("Unsupported protocol");
|
||||
}
|
||||
|
||||
/**
|
||||
* clean up template data on staging area
|
||||
* @param newTemplate: The template on the secondary storage that needs to be cleaned up
|
||||
*/
|
||||
protected void cleanupStagingNfs(TemplateObjectTO newTemplate) {
|
||||
try {
|
||||
DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
|
||||
execute(deleteCommand);
|
||||
} catch (Exception e) {
|
||||
s_logger.debug("Failed to clean up staging area:", e);
|
||||
}
|
||||
}
|
||||
|
||||
protected Answer copyFromNfsToImage(CopyCommand cmd) {
|
||||
|
|
@ -759,22 +746,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
String container = "T-" + cmd.getId();
|
||||
String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null);
|
||||
|
||||
long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName()));
|
||||
long size = file.length();
|
||||
String uniqueName = cmd.getName();
|
||||
|
||||
//put metda file
|
||||
File uniqDir = _storage.createUniqDir();
|
||||
String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
|
||||
_storage.create(uniqDir.getAbsolutePath(), "template.properties");
|
||||
File metaFile = new File(metaFileName);
|
||||
FileWriter writer = new FileWriter(metaFile);
|
||||
BufferedWriter bufferWriter = new BufferedWriter(writer);
|
||||
bufferWriter.write("uniquename=" + cmd.getName());
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("filename=" + fileName);
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("size=" + file.length());
|
||||
bufferWriter.close();
|
||||
writer.close();
|
||||
String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
|
||||
_storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
|
||||
|
||||
SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
|
||||
File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size, virtualSize);
|
||||
|
||||
SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp);
|
||||
metaFile.delete();
|
||||
uniqDir.delete();
|
||||
String md5sum = null;
|
||||
|
|
@ -785,7 +768,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
}
|
||||
|
||||
DownloadAnswer answer =
|
||||
new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum);
|
||||
new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum);
|
||||
return answer;
|
||||
} catch (IOException e) {
|
||||
s_logger.debug("Failed to register template into swift", e);
|
||||
|
|
@ -942,6 +925,118 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
}
|
||||
}
|
||||
|
||||
/***
|
||||
*This method will create a file using the filenName and metaFileName.
|
||||
*That file will contain the given attributes (unique name, file name, size, and virtualSize).
|
||||
*
|
||||
* @param metaFileName : The path of the metadata file
|
||||
* @param filename :attribute: Filename of the template
|
||||
* @param uniqueName :attribute: Unique name of the template
|
||||
* @param size :attribute: physical size of the template
|
||||
* @param virtualSize :attribute: virtual size of the template
|
||||
* @return File representing the metadata file
|
||||
* @throws IOException
|
||||
*/
|
||||
|
||||
protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException {
|
||||
File metaFile = new File(metaFileName);
|
||||
FileWriter writer = new FileWriter(metaFile);
|
||||
BufferedWriter bufferWriter = new BufferedWriter(writer);
|
||||
bufferWriter.write("uniquename=" + uniqueName);
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("filename=" + filename);
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("size=" + size);
|
||||
bufferWriter.write("\n");
|
||||
bufferWriter.write("virtualsize=" + virtualSize);
|
||||
bufferWriter.close();
|
||||
writer.close();
|
||||
return metaFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a template.properties for Swift with its correct unique name
|
||||
*
|
||||
* @param swift The swift object
|
||||
* @param srcFile Source file on the staging NFS
|
||||
* @param containerName Destination container
|
||||
* @return true on successful write
|
||||
*/
|
||||
protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException {
|
||||
|
||||
String uniqueName = FilenameUtils.getBaseName(srcFile.getName());
|
||||
|
||||
File uniqDir = _storage.createUniqDir();
|
||||
String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp;
|
||||
_storage.create(uniqDir.getAbsolutePath(), _tmpltpp);
|
||||
|
||||
long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()));
|
||||
|
||||
File metaFile = swiftWriteMetadataFile(metaFileName,
|
||||
uniqueName,
|
||||
srcFile.getName(),
|
||||
srcFile.length(),
|
||||
virtualSize);
|
||||
|
||||
SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp);
|
||||
metaFile.delete();
|
||||
uniqDir.delete();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies data from NFS and uploads it into a Swift container
|
||||
*
|
||||
* @param cmd CopyComand
|
||||
* @return CopyCmdAnswer
|
||||
*/
|
||||
protected Answer copyFromNfsToSwift(CopyCommand cmd) {
|
||||
|
||||
final DataTO srcData = cmd.getSrcTO();
|
||||
final DataTO destData = cmd.getDestTO();
|
||||
|
||||
DataStoreTO srcDataStore = srcData.getDataStore();
|
||||
NfsTO srcStore = (NfsTO)srcDataStore;
|
||||
DataStoreTO destDataStore = destData.getDataStore();
|
||||
File srcFile = getFile(srcData.getPath(), srcStore.getUrl());
|
||||
|
||||
SwiftTO swift = (SwiftTO)destDataStore;
|
||||
|
||||
try {
|
||||
|
||||
String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
|
||||
String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName());
|
||||
|
||||
|
||||
DataTO retObj = null;
|
||||
if (destData.getObjectType() == DataObjectType.TEMPLATE) {
|
||||
swiftUploadMetadataFile(swift, srcFile, containerName);
|
||||
TemplateObjectTO newTemplate = new TemplateObjectTO();
|
||||
newTemplate.setPath(swiftPath);
|
||||
newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
|
||||
newTemplate.setPhysicalSize(srcFile.length());
|
||||
newTemplate.setFormat(getTemplateFormat(srcFile.getName()));
|
||||
retObj = newTemplate;
|
||||
} else if (destData.getObjectType() == DataObjectType.VOLUME) {
|
||||
VolumeObjectTO newVol = new VolumeObjectTO();
|
||||
newVol.setPath(containerName);
|
||||
newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
|
||||
retObj = newVol;
|
||||
} else if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
|
||||
SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
|
||||
newSnapshot.setPath(containerName);
|
||||
retObj = newSnapshot;
|
||||
}
|
||||
|
||||
return new CopyCmdAnswer(retObj);
|
||||
|
||||
} catch (Exception e) {
|
||||
s_logger.error("failed to upload " + srcData.getPath(), e);
|
||||
return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString());
|
||||
}
|
||||
}
|
||||
|
||||
String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
|
||||
Script command = new Script("/bin/bash", s_logger);
|
||||
command.add("-c");
|
||||
|
|
@ -1458,13 +1553,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
Map<String, TemplateProp> tmpltInfos = new HashMap<String, TemplateProp>();
|
||||
for (String container : containers) {
|
||||
if (container.startsWith("T-")) {
|
||||
String[] files = SwiftUtil.list(swift, container, "template.properties");
|
||||
String[] files = SwiftUtil.list(swift, container, _tmpltpp);
|
||||
if (files.length != 1) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
File tempFile = File.createTempFile("template", ".tmp");
|
||||
File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties");
|
||||
File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + _tmpltpp);
|
||||
if (tmpFile == null) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1779,7 +1874,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S
|
|||
} else {
|
||||
boolean found = false;
|
||||
for (File f : tmpltFiles) {
|
||||
if (!found && f.getName().equals("template.properties")) {
|
||||
if (!found && f.getName().equals(_tmpltpp)) {
|
||||
found = true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,29 +18,6 @@
|
|||
*/
|
||||
package org.apache.cloudstack.storage.resource;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
|
||||
import org.apache.cloudstack.storage.command.CopyCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadCommand;
|
||||
import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
||||
|
||||
import com.cloud.agent.api.storage.DownloadAnswer;
|
||||
import com.cloud.agent.api.storage.ListTemplateAnswer;
|
||||
import com.cloud.agent.api.storage.ListTemplateCommand;
|
||||
|
|
@ -51,7 +28,28 @@ import com.cloud.storage.DataStoreRole;
|
|||
import com.cloud.storage.Storage;
|
||||
import com.cloud.utils.PropertiesUtil;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import junit.framework.Assert;
|
||||
import junit.framework.TestCase;
|
||||
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
|
||||
import org.apache.cloudstack.storage.command.CopyCommand;
|
||||
import org.apache.cloudstack.storage.command.DownloadCommand;
|
||||
import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.UUID;
|
||||
|
||||
@Ignore
|
||||
public class LocalNfsSecondaryStorageResourceTest extends TestCase {
|
||||
private static Map<String, Object> testParams;
|
||||
|
||||
|
|
|
|||
|
|
@ -18,91 +18,67 @@
|
|||
*/
|
||||
package org.apache.cloudstack.storage.resource;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import javax.naming.ConfigurationException;
|
||||
|
||||
import junit.framework.Assert;
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import com.cloud.test.TestAppender;
|
||||
import org.apache.cloudstack.storage.command.DeleteCommand;
|
||||
import org.apache.cloudstack.storage.to.TemplateObjectTO;
|
||||
import org.apache.log4j.Level;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mockito;
|
||||
import org.powermock.api.mockito.PowerMockito;
|
||||
import org.powermock.core.classloader.annotations.PrepareForTest;
|
||||
import org.powermock.modules.junit4.PowerMockRunner;
|
||||
|
||||
import com.cloud.utils.PropertiesUtil;
|
||||
import com.cloud.utils.exception.CloudRuntimeException;
|
||||
import java.io.BufferedWriter;
|
||||
import java.io.FileWriter;
|
||||
import java.io.StringWriter;
|
||||
|
||||
public class NfsSecondaryStorageResourceTest extends TestCase {
|
||||
private static Map<String, Object> testParams;
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.spy;
|
||||
|
||||
private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName());
|
||||
@RunWith(PowerMockRunner.class)
|
||||
public class NfsSecondaryStorageResourceTest {
|
||||
|
||||
NfsSecondaryStorageResource resource;
|
||||
private NfsSecondaryStorageResource resource;
|
||||
|
||||
@Before
|
||||
@Override
|
||||
public void setUp() throws ConfigurationException {
|
||||
s_logger.setLevel(Level.ALL);
|
||||
public void setUp() {
|
||||
resource = new NfsSecondaryStorageResource();
|
||||
resource.setInSystemVM(true);
|
||||
testParams = PropertiesUtil.toMap(loadProperties());
|
||||
resource.configureStorageLayerClass(testParams);
|
||||
Object testLocalRoot = testParams.get("testLocalRoot");
|
||||
if (testLocalRoot != null) {
|
||||
resource.setParentPath((String)testLocalRoot);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMount() throws Exception {
|
||||
String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar";
|
||||
URI sampleUri = new URI(sampleUriStr);
|
||||
@PrepareForTest(NfsSecondaryStorageResource.class)
|
||||
public void testSwiftWriteMetadataFile() throws Exception {
|
||||
String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000";
|
||||
|
||||
s_logger.info("Check HostIp parsing");
|
||||
String hostIpStr = resource.getUriHostIp(sampleUri);
|
||||
Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr);
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
BufferedWriter bufferWriter = new BufferedWriter(stringWriter);
|
||||
PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter);
|
||||
|
||||
s_logger.info("Check option parsing");
|
||||
String expected = "user=administrator,password=1pass@word1,foo=bar,";
|
||||
String actualOpts = resource.parseCifsMountOptions(sampleUri);
|
||||
Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts);
|
||||
resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000);
|
||||
|
||||
// attempt a configured mount
|
||||
final Map<String, Object> params = PropertiesUtil.toMap(loadProperties());
|
||||
String sampleMount = (String)params.get("testCifsMount");
|
||||
if (!sampleMount.isEmpty()) {
|
||||
s_logger.info("functional test, mount " + sampleMount);
|
||||
URI realMntUri = new URI(sampleMount);
|
||||
String mntSubDir = resource.mountUri(realMntUri);
|
||||
s_logger.info("functional test, umount " + mntSubDir);
|
||||
resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri);
|
||||
} else {
|
||||
s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test");
|
||||
}
|
||||
Assert.assertEquals(expected, stringWriter.toString());
|
||||
}
|
||||
|
||||
public static Properties loadProperties() throws ConfigurationException {
|
||||
Properties properties = new Properties();
|
||||
final File file = PropertiesUtil.findConfigFile("agent.properties");
|
||||
if (file == null) {
|
||||
throw new ConfigurationException("Unable to find agent.properties.");
|
||||
}
|
||||
s_logger.info("agent.properties found at " + file.getAbsolutePath());
|
||||
try(FileInputStream fs = new FileInputStream(file);) {
|
||||
properties.load(fs);
|
||||
} catch (final FileNotFoundException ex) {
|
||||
throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex);
|
||||
} catch (final IOException ex) {
|
||||
throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex);
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
@Test
|
||||
public void testCleanupStagingNfs() throws Exception{
|
||||
|
||||
NfsSecondaryStorageResource spyResource = spy(resource);
|
||||
RuntimeException exception = new RuntimeException();
|
||||
doThrow(exception).when(spyResource).execute(any(DeleteCommand.class));
|
||||
TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class);
|
||||
|
||||
TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder();
|
||||
appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:");
|
||||
TestAppender testLogAppender = appenderBuilder.build();
|
||||
TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender);
|
||||
|
||||
spyResource.cleanupStagingNfs(mockTemplate);
|
||||
|
||||
testLogAppender.assertMessagesLogged();
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue