diff --git a/agent/bindir/libvirtqemuhook.in b/agent/bindir/libvirtqemuhook.in index 3f290c6b81b..9fbe037d1c1 100755 --- a/agent/bindir/libvirtqemuhook.in +++ b/agent/bindir/libvirtqemuhook.in @@ -26,6 +26,8 @@ def isOldStyleBridge(brName): else: return False def isNewStyleBridge(brName): + if brName.startswith('brvx-'): + return False if re.match(r"br(\w+)-(\d+)", brName) == None: return False else: diff --git a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java index b1d6127e243..5117b7cb84f 100644 --- a/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java +++ b/engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java @@ -156,6 +156,7 @@ public class ObjectInDataStoreManagerImpl implements ObjectInDataStoreManager { // template.properties // there } + ts.setInstallPath(installPath); ts.setState(ObjectInDataStoreStateMachine.State.Allocated); ts = templateDataStoreDao.persist(ts); diff --git a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java index fc448a347a4..3332393f652 100644 --- a/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java +++ b/server/src/com/cloud/network/router/VirtualNetworkApplianceManagerImpl.java @@ -1531,6 +1531,9 @@ Configurable, StateListener> expectedPatternResults; + private TestAppender(final Map> expectedPatterns) { + super(); + expectedPatternResults = ImmutableMap.copyOf(expectedPatterns); + } + protected void append(LoggingEvent loggingEvent) { + checkArgument(loggingEvent != null, "append requires a non-null loggingEvent"); + final Level level = loggingEvent.getLevel(); + checkState(expectedPatternResults.containsKey(level), "level " + level + " not supported by append"); + for (final PatternResult patternResult : expectedPatternResults.get(level)) { + if (patternResult.getPattern().matcher(loggingEvent.getRenderedMessage()).matches()) { + patternResult.markFound(); + } + } + } + + public void close() { +// Do nothing ... + } + public boolean requiresLayout() { + return false; + } + public void assertMessagesLogged() { + final List unloggedPatterns = new ArrayList<>(); + for (final Map.Entry> expectedPatternResult : expectedPatternResults.entrySet()) { + for (final PatternResult patternResults : expectedPatternResult.getValue()) { + if (!patternResults.isFound()) { + unloggedPatterns.add(format("%1$s was not logged for level %2$s", + patternResults.getPattern().toString(), expectedPatternResult.getKey())); + } + } + } + if (!unloggedPatterns.isEmpty()) { + //Raise an assert + Assert.isTrue(false, Joiner.on(",").join(unloggedPatterns)); + } + } + + private static final class PatternResult { + private final Pattern pattern; + private boolean foundFlag = false; + private PatternResult(Pattern pattern) { + super(); + this.pattern = pattern; + } + public Pattern getPattern() { + return pattern; + } + public void markFound() { + // This operation is thread-safe because the value will only ever be switched from false to true. Therefore, + // multiple threads mutating the value for a pattern will not corrupt the value ... + foundFlag = true; + } + public boolean isFound() { + return foundFlag; + } + @Override + public boolean equals(Object thatObject) { + if (this == thatObject) { + return true; + } + if (thatObject == null || getClass() != thatObject.getClass()) { + return false; + } + PatternResult thatPatternResult = (PatternResult) thatObject; + return foundFlag == thatPatternResult.foundFlag && + Objects.equal(pattern, thatPatternResult.pattern); + } + @Override + public int hashCode() { + return Objects.hashCode(pattern, foundFlag); + } + @Override + public String toString() { + return format("Pattern Result [ pattern: %1$s, markFound: %2$s ]", pattern.toString(), foundFlag); + } + } + + public static final class TestAppenderBuilder { + private final Map> expectedPatterns; + public TestAppenderBuilder() { + super(); + expectedPatterns = new HashMap<>(); + expectedPatterns.put(ALL, new HashSet()); + expectedPatterns.put(DEBUG, new HashSet()); + expectedPatterns.put(ERROR, new HashSet()); + expectedPatterns.put(FATAL, new HashSet()); + expectedPatterns.put(INFO, new HashSet()); + expectedPatterns.put(OFF, new HashSet()); + } + public TestAppenderBuilder addExpectedPattern(final Level level, final String pattern) { + checkArgument(level != null, "addExpectedPattern requires a non-null level"); + checkArgument(!isNullOrEmpty(pattern), "addExpectedPattern requires a non-blank pattern"); + checkState(expectedPatterns.containsKey(level), "level " + level + " is not supported by " + getClass().getName()); + expectedPatterns.get(level).add(new PatternResult(Pattern.compile(pattern))); + return this; + } + public TestAppender build() { + return new TestAppender(expectedPatterns); + } + } + /** + * + * Attaches a {@link TestAppender} to a {@link Logger} and ensures that it is the only + * test appender attached to the logger. + * + * @param logger The logger which will be monitored by the test + * @param testAppender The test appender to attach to {@code logger} + */ + public static void safeAddAppender(Logger logger, TestAppender testAppender) { + logger.removeAppender(APPENDER_NAME); + logger.addAppender(testAppender); + } +} \ No newline at end of file diff --git a/services/secondary-storage/server/pom.xml b/services/secondary-storage/server/pom.xml index f3d01762079..50155cfb610 100644 --- a/services/secondary-storage/server/pom.xml +++ b/services/secondary-storage/server/pom.xml @@ -26,9 +26,6 @@ 4.9.0-SNAPSHOT ../pom.xml - - true - log4j diff --git a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 08eab337c9b..e7a9076f148 100644 --- a/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -16,8 +16,8 @@ // under the License. package org.apache.cloudstack.storage.resource; -import static com.cloud.utils.StringUtils.join; import static com.cloud.utils.storage.S3.S3Utils.putFile; +import static com.cloud.utils.StringUtils.join; import static java.lang.String.format; import static java.util.Arrays.asList; import static org.apache.commons.lang.StringUtils.substringAfterLast; @@ -46,25 +46,27 @@ import java.util.UUID; import javax.naming.ConfigurationException; -import org.apache.cloudstack.framework.security.keystore.KeystoreManager; -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DeleteCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import com.cloud.exception.InvalidParameterValueException; +import com.cloud.storage.Storage; +import com.cloud.storage.template.TemplateConstants; +import com.cloud.utils.EncryptionUtil; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.Channel; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.ChannelPipeline; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.http.HttpContentCompressor; +import io.netty.handler.codec.http.HttpRequestDecoder; +import io.netty.handler.codec.http.HttpResponseEncoder; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; -import org.apache.cloudstack.storage.command.UploadStatusAnswer; -import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; -import org.apache.cloudstack.storage.command.UploadStatusCommand; -import org.apache.cloudstack.storage.template.DownloadManager; -import org.apache.cloudstack.storage.template.DownloadManagerImpl; -import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; import org.apache.cloudstack.storage.template.UploadEntity; -import org.apache.cloudstack.storage.template.UploadManager; -import org.apache.cloudstack.storage.template.UploadManagerImpl; -import org.apache.cloudstack.storage.to.SnapshotObjectTO; -import org.apache.cloudstack.storage.to.TemplateObjectTO; -import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.imagestore.ImageStoreUtil; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; @@ -78,10 +80,27 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; -import org.joda.time.DateTime; -import org.joda.time.format.ISODateTimeFormat; import com.amazonaws.services.s3.model.S3ObjectSummary; + +import org.apache.cloudstack.framework.security.keystore.KeystoreManager; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.command.DownloadProgressCommand; +import org.apache.cloudstack.storage.command.UploadStatusAnswer; +import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; +import org.apache.cloudstack.storage.command.UploadStatusCommand; +import org.apache.cloudstack.storage.template.DownloadManager; +import org.apache.cloudstack.storage.template.DownloadManagerImpl; +import org.apache.cloudstack.storage.template.DownloadManagerImpl.ZfsPathParser; +import org.apache.cloudstack.storage.template.UploadManager; +import org.apache.cloudstack.storage.template.UploadManagerImpl; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; + import com.cloud.agent.api.Answer; import com.cloud.agent.api.CheckHealthAnswer; import com.cloud.agent.api.CheckHealthCommand; @@ -116,13 +135,11 @@ import com.cloud.agent.api.to.NfsTO; import com.cloud.agent.api.to.S3TO; import com.cloud.agent.api.to.SwiftTO; import com.cloud.exception.InternalErrorException; -import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.Host.Type; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.resource.ServerResourceBase; import com.cloud.storage.DataStoreRole; -import com.cloud.storage.Storage; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.StorageLayer; import com.cloud.storage.VMTemplateStorageResourceAssoc; @@ -132,40 +149,24 @@ import com.cloud.storage.template.Processor.FormatInfo; import com.cloud.storage.template.QCOW2Processor; import com.cloud.storage.template.RawImageProcessor; import com.cloud.storage.template.TARProcessor; -import com.cloud.storage.template.TemplateConstants; import com.cloud.storage.template.TemplateLocation; import com.cloud.storage.template.TemplateProp; import com.cloud.storage.template.VhdProcessor; import com.cloud.storage.template.VmdkProcessor; -import com.cloud.utils.EncryptionUtil; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.storage.S3.S3Utils; import com.cloud.utils.SwiftUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.net.NetUtils; import com.cloud.utils.script.OutputInterpreter; import com.cloud.utils.script.Script; -import com.cloud.utils.storage.S3.S3Utils; import com.cloud.vm.SecondaryStorageVm; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.SocketChannel; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.codec.http.HttpContentCompressor; -import io.netty.handler.codec.http.HttpRequestDecoder; -import io.netty.handler.codec.http.HttpResponseEncoder; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; +import org.joda.time.DateTime; +import org.joda.time.format.ISODateTimeFormat; public class NfsSecondaryStorageResource extends ServerResourceBase implements SecondaryStorageResource { - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); + public static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResource.class); private static final String TEMPLATE_ROOT_DIR = "template/tmpl"; private static final String VOLUME_ROOT_DIR = "volumes"; @@ -492,10 +493,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String destFileFullPath = destFile.getAbsolutePath() + File.separator + fileName; s_logger.debug("copy snapshot " + srcFile.getAbsolutePath() + " to template " + destFileFullPath); Script.runSimpleBashScript("cp " + srcFile.getAbsolutePath() + " " + destFileFullPath); - String metaFileName = destFile.getAbsolutePath() + File.separator + "template.properties"; + String metaFileName = destFile.getAbsolutePath() + File.separator + _tmpltpp; File metaFile = new File(metaFileName); try { - _storage.create(destFile.getAbsolutePath(), "template.properties"); + _storage.create(destFile.getAbsolutePath(), _tmpltpp); try ( // generate template.properties file FileWriter writer = new FileWriter(metaFile); BufferedWriter bufferWriter = new BufferedWriter(writer); @@ -590,32 +591,14 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return answer; } s_logger.debug("starting copy template to swift"); - DataTO newTemplate = answer.getNewData(); - File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl(), _nfsVersion); - SwiftTO swift = (SwiftTO)destDataStore; - String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); - String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName()); - //upload template.properties - File properties = new File(templateFile.getParent() + File.separator + _tmpltpp); - if (properties.exists()) { - SwiftUtil.putObject(swift, properties, containterName, _tmpltpp); - } + TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData(); + newTemplate.setDataStore(srcDataStore); + CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence()); + Answer result = copyFromNfsToSwift(newCpyCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(newTemplate); - execute(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } + cleanupStagingNfs(newTemplate); + return result; - TemplateObjectTO template = new TemplateObjectTO(); - template.setPath(swiftPath); - template.setSize(templateFile.length()); - template.setPhysicalSize(template.getSize()); - SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData; - template.setFormat(snapshot.getVolume().getFormat()); - return new CopyCmdAnswer(template); } else if (destDataStore instanceof S3TO) { //create template on the same data store CopyCmdAnswer answer = @@ -628,18 +611,27 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S newTemplate.setDataStore(srcDataStore); CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence()); Answer result = copyFromNfsToS3(newCpyCmd); - //clean up template data on staging area - try { - DeleteCommand deleteCommand = new DeleteCommand(newTemplate); - execute(deleteCommand); - } catch (Exception e) { - s_logger.debug("Failed to clean up staging area:", e); - } + + cleanupStagingNfs(newTemplate); + return result; } } - s_logger.debug("Failed to create templat from snapshot"); - return new CopyCmdAnswer("Unsupported prototcol"); + s_logger.debug("Failed to create template from snapshot"); + return new CopyCmdAnswer("Unsupported protocol"); + } + + /** + * clean up template data on staging area + * @param newTemplate: The template on the secondary storage that needs to be cleaned up + */ + protected void cleanupStagingNfs(TemplateObjectTO newTemplate) { + try { + DeleteCommand deleteCommand = new DeleteCommand(newTemplate); + execute(deleteCommand); + } catch (Exception e) { + s_logger.debug("Failed to clean up staging area:", e); + } } protected Answer copyFromNfsToImage(CopyCommand cmd) { @@ -750,22 +742,18 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S String container = "T-" + cmd.getId(); String swiftPath = SwiftUtil.putObject(swiftTO, file, container, null); + long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName())); + long size = file.length(); + String uniqueName = cmd.getName(); + //put metda file File uniqDir = _storage.createUniqDir(); - String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties"; - _storage.create(uniqDir.getAbsolutePath(), "template.properties"); - File metaFile = new File(metaFileName); - FileWriter writer = new FileWriter(metaFile); - BufferedWriter bufferWriter = new BufferedWriter(writer); - bufferWriter.write("uniquename=" + cmd.getName()); - bufferWriter.write("\n"); - bufferWriter.write("filename=" + fileName); - bufferWriter.write("\n"); - bufferWriter.write("size=" + file.length()); - bufferWriter.close(); - writer.close(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); - SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties"); + File metaFile = swiftWriteMetadataFile(metaFileName, uniqueName, fileName, size, virtualSize); + + SwiftUtil.putObject(swiftTO, metaFile, container, _tmpltpp); metaFile.delete(); uniqDir.delete(); String md5sum = null; @@ -776,7 +764,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } DownloadAnswer answer = - new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum); + new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum); return answer; } catch (IOException e) { s_logger.debug("Failed to register template into swift", e); @@ -933,6 +921,118 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + /*** + *This method will create a file using the filenName and metaFileName. + *That file will contain the given attributes (unique name, file name, size, and virtualSize). + * + * @param metaFileName : The path of the metadata file + * @param filename :attribute: Filename of the template + * @param uniqueName :attribute: Unique name of the template + * @param size :attribute: physical size of the template + * @param virtualSize :attribute: virtual size of the template + * @return File representing the metadata file + * @throws IOException + */ + + protected File swiftWriteMetadataFile(String metaFileName, String uniqueName, String filename, long size, long virtualSize) throws IOException { + File metaFile = new File(metaFileName); + FileWriter writer = new FileWriter(metaFile); + BufferedWriter bufferWriter = new BufferedWriter(writer); + bufferWriter.write("uniquename=" + uniqueName); + bufferWriter.write("\n"); + bufferWriter.write("filename=" + filename); + bufferWriter.write("\n"); + bufferWriter.write("size=" + size); + bufferWriter.write("\n"); + bufferWriter.write("virtualsize=" + virtualSize); + bufferWriter.close(); + writer.close(); + return metaFile; + } + + /** + * Creates a template.properties for Swift with its correct unique name + * + * @param swift The swift object + * @param srcFile Source file on the staging NFS + * @param containerName Destination container + * @return true on successful write + */ + protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException { + + String uniqueName = FilenameUtils.getBaseName(srcFile.getName()); + + File uniqDir = _storage.createUniqDir(); + String metaFileName = uniqDir.getAbsolutePath() + File.separator + _tmpltpp; + _storage.create(uniqDir.getAbsolutePath(), _tmpltpp); + + long virtualSize = getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())); + + File metaFile = swiftWriteMetadataFile(metaFileName, + uniqueName, + srcFile.getName(), + srcFile.length(), + virtualSize); + + SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp); + metaFile.delete(); + uniqDir.delete(); + + return true; + } + + /** + * Copies data from NFS and uploads it into a Swift container + * + * @param cmd CopyComand + * @return CopyCmdAnswer + */ + protected Answer copyFromNfsToSwift(CopyCommand cmd) { + + final DataTO srcData = cmd.getSrcTO(); + final DataTO destData = cmd.getDestTO(); + + DataStoreTO srcDataStore = srcData.getDataStore(); + NfsTO srcStore = (NfsTO)srcDataStore; + DataStoreTO destDataStore = destData.getDataStore(); + File srcFile = getFile(srcData.getPath(), srcStore.getUrl(), _nfsVersion); + + SwiftTO swift = (SwiftTO)destDataStore; + + try { + + String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId()); + String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName()); + + + DataTO retObj = null; + if (destData.getObjectType() == DataObjectType.TEMPLATE) { + swiftUploadMetadataFile(swift, srcFile, containerName); + TemplateObjectTO newTemplate = new TemplateObjectTO(); + newTemplate.setPath(swiftPath); + newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + newTemplate.setPhysicalSize(srcFile.length()); + newTemplate.setFormat(getTemplateFormat(srcFile.getName())); + retObj = newTemplate; + } else if (destData.getObjectType() == DataObjectType.VOLUME) { + VolumeObjectTO newVol = new VolumeObjectTO(); + newVol.setPath(containerName); + newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName()))); + retObj = newVol; + } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) { + SnapshotObjectTO newSnapshot = new SnapshotObjectTO(); + newSnapshot.setPath(containerName); + retObj = newSnapshot; + } + + return new CopyCmdAnswer(retObj); + + } catch (Exception e) { + s_logger.error("failed to upload " + srcData.getPath(), e); + return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString()); + } + } + String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) { Script command = new Script("/bin/bash", s_logger); command.add("-c"); @@ -1450,13 +1550,13 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S Map tmpltInfos = new HashMap(); for (String container : containers) { if (container.startsWith("T-")) { - String[] files = SwiftUtil.list(swift, container, "template.properties"); + String[] files = SwiftUtil.list(swift, container, _tmpltpp); if (files.length != 1) { continue; } try { File tempFile = File.createTempFile("template", ".tmp"); - File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + "template.properties"); + File tmpFile = SwiftUtil.getObject(swift, tempFile, container + File.separator + _tmpltpp); if (tmpFile == null) { continue; } @@ -1771,7 +1871,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } else { boolean found = false; for (File f : tmpltFiles) { - if (!found && f.getName().equals("template.properties")) { + if (!found && f.getName().equals(_tmpltpp)) { found = true; } diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java index 52bde6a11da..b33ce3b7474 100644 --- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java @@ -18,29 +18,6 @@ */ package org.apache.cloudstack.storage.resource; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import javax.naming.ConfigurationException; - -import junit.framework.Assert; -import junit.framework.TestCase; - -import org.apache.log4j.Logger; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import org.apache.cloudstack.storage.command.CopyCmdAnswer; -import org.apache.cloudstack.storage.command.CopyCommand; -import org.apache.cloudstack.storage.command.DownloadCommand; -import org.apache.cloudstack.storage.to.TemplateObjectTO; - import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.ListTemplateAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; @@ -51,7 +28,28 @@ import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; +import junit.framework.Assert; +import junit.framework.TestCase; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.apache.log4j.Logger; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockito.Mockito; +import javax.naming.ConfigurationException; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; + +@Ignore public class LocalNfsSecondaryStorageResourceTest extends TestCase { private static Map testParams; diff --git a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java index 9da5c53a903..e437a0f3b64 100644 --- a/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java +++ b/services/secondary-storage/server/test/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResourceTest.java @@ -18,91 +18,67 @@ */ package org.apache.cloudstack.storage.resource; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.net.URI; -import java.util.Map; -import java.util.Properties; - -import javax.naming.ConfigurationException; - -import junit.framework.Assert; -import junit.framework.TestCase; - +import com.cloud.test.TestAppender; +import org.apache.cloudstack.storage.command.DeleteCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.log4j.Level; -import org.apache.log4j.Logger; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; -import com.cloud.utils.PropertiesUtil; -import com.cloud.utils.exception.CloudRuntimeException; +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.StringWriter; -public class NfsSecondaryStorageResourceTest extends TestCase { - private static Map testParams; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; - private static final Logger s_logger = Logger.getLogger(NfsSecondaryStorageResourceTest.class.getName()); +@RunWith(PowerMockRunner.class) +public class NfsSecondaryStorageResourceTest { - NfsSecondaryStorageResource resource; + private NfsSecondaryStorageResource resource; @Before - @Override - public void setUp() throws ConfigurationException { - s_logger.setLevel(Level.ALL); + public void setUp() { resource = new NfsSecondaryStorageResource(); - resource.setInSystemVM(true); - testParams = PropertiesUtil.toMap(loadProperties()); - resource.configureStorageLayerClass(testParams); - Object testLocalRoot = testParams.get("testLocalRoot"); - if (testLocalRoot != null) { - resource.setParentPath((String)testLocalRoot); - } } @Test - public void testMount() throws Exception { - String sampleUriStr = "cifs://192.168.1.128/CSHV3?user=administrator&password=1pass%40word1&foo=bar"; - URI sampleUri = new URI(sampleUriStr); + @PrepareForTest(NfsSecondaryStorageResource.class) + public void testSwiftWriteMetadataFile() throws Exception { + String expected = "uniquename=test\nfilename=testfile\nsize=100\nvirtualsize=1000"; - s_logger.info("Check HostIp parsing"); - String hostIpStr = resource.getUriHostIp(sampleUri); - Assert.assertEquals("Expected host IP " + sampleUri.getHost() + " and actual host IP " + hostIpStr + " differ.", sampleUri.getHost(), hostIpStr); + StringWriter stringWriter = new StringWriter(); + BufferedWriter bufferWriter = new BufferedWriter(stringWriter); + PowerMockito.whenNew(BufferedWriter.class).withArguments(any(FileWriter.class)).thenReturn(bufferWriter); - s_logger.info("Check option parsing"); - String expected = "user=administrator,password=1pass@word1,foo=bar,"; - String actualOpts = resource.parseCifsMountOptions(sampleUri); - Assert.assertEquals("Options should be " + expected + " and not " + actualOpts, expected, actualOpts); + resource.swiftWriteMetadataFile("testfile", "test", "testfile", 100, 1000); - // attempt a configured mount - final Map params = PropertiesUtil.toMap(loadProperties()); - String sampleMount = (String)params.get("testCifsMount"); - if (!sampleMount.isEmpty()) { - s_logger.info("functional test, mount " + sampleMount); - URI realMntUri = new URI(sampleMount); - String mntSubDir = resource.mountUri(realMntUri, null); - s_logger.info("functional test, umount " + mntSubDir); - resource.umount(resource.getMountingRoot() + mntSubDir, realMntUri); - } else { - s_logger.info("no entry for testCifsMount in " + "./conf/agent.properties - skip functional test"); - } + Assert.assertEquals(expected, stringWriter.toString()); } - public static Properties loadProperties() throws ConfigurationException { - Properties properties = new Properties(); - final File file = PropertiesUtil.findConfigFile("agent.properties"); - if (file == null) { - throw new ConfigurationException("Unable to find agent.properties."); - } - s_logger.info("agent.properties found at " + file.getAbsolutePath()); - try(FileInputStream fs = new FileInputStream(file);) { - properties.load(fs); - } catch (final FileNotFoundException ex) { - throw new CloudRuntimeException("Cannot find the file: " + file.getAbsolutePath(), ex); - } catch (final IOException ex) { - throw new CloudRuntimeException("IOException in reading " + file.getAbsolutePath(), ex); - } - return properties; - } + @Test + public void testCleanupStagingNfs() throws Exception{ + NfsSecondaryStorageResource spyResource = spy(resource); + RuntimeException exception = new RuntimeException(); + doThrow(exception).when(spyResource).execute(any(DeleteCommand.class)); + TemplateObjectTO mockTemplate = Mockito.mock(TemplateObjectTO.class); + + TestAppender.TestAppenderBuilder appenderBuilder = new TestAppender.TestAppenderBuilder(); + appenderBuilder.addExpectedPattern(Level.DEBUG, "Failed to clean up staging area:"); + TestAppender testLogAppender = appenderBuilder.build(); + TestAppender.safeAddAppender(NfsSecondaryStorageResource.s_logger, testLogAppender); + + spyResource.cleanupStagingNfs(mockTemplate); + + testLogAppender.assertMessagesLogged(); + + } } diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py index ce490aa9db4..9ccb768d14c 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsDatabag.py @@ -154,3 +154,7 @@ class CsCmdLine(CsDataBag): return self.idata()['useextdns'] return False + def get_advert_int(self): + if 'advert_int' in self.idata(): + return self.idata()['advert_int'] + return 1 diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py index 7829c0a99e7..78ad8597f8b 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsFile.py @@ -113,6 +113,7 @@ class CsFile: self.new_config[sind:eind] = content def greplace(self, search, replace): + logging.debug("Searching for %s and replacing with %s" % (search, replace)) self.new_config = [w.replace(search, replace) for w in self.new_config] def search(self, search, replace): diff --git a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py index f1ab5f785d9..f8d2bc25665 100755 --- a/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py +++ b/systemvm/patches/debian/config/opt/cloud/bin/cs/CsRedundant.py @@ -138,6 +138,9 @@ class CsRedundant(object): " router_id ", " router_id %s" % self.cl.get_name()) keepalived_conf.search( " interface ", " interface %s" % guest.get_device()) + keepalived_conf.search( + " advert_int ", " advert_int %s" % self.cl.get_advert_int()) + keepalived_conf.greplace("[RROUTER_BIN_PATH]", self.CS_ROUTER_DIR) keepalived_conf.section("authentication {", "}", [ " auth_type AH \n", " auth_pass %s\n" % self.cl.get_router_password()])