diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index ebe7590fd5f..025d9e97d95 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -374,6 +374,7 @@ public class EventTypes { // Primary storage pool public static final String EVENT_ENABLE_PRIMARY_STORAGE = "ENABLE.PS"; public static final String EVENT_DISABLE_PRIMARY_STORAGE = "DISABLE.PS"; + public static final String EVENT_SYNC_STORAGE_POOL = "SYNC.STORAGE.POOL"; // VPN public static final String EVENT_REMOTE_ACCESS_VPN_CREATE = "VPN.REMOTE.ACCESS.CREATE"; diff --git a/api/src/main/java/com/cloud/storage/StorageService.java b/api/src/main/java/com/cloud/storage/StorageService.java index 207fc8f0cd7..4b18739b55d 100644 --- a/api/src/main/java/com/cloud/storage/StorageService.java +++ b/api/src/main/java/com/cloud/storage/StorageService.java @@ -27,6 +27,7 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import com.cloud.exception.DiscoveryException; import com.cloud.exception.InsufficientCapacityException; @@ -104,4 +105,6 @@ public interface StorageService { ImageStore updateImageStoreStatus(Long id, Boolean readonly); + StoragePool syncStoragePool(SyncStoragePoolCmd cmd); + } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/SyncStoragePoolCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/SyncStoragePoolCmd.java new file mode 100644 index 00000000000..9761e330a4c --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/SyncStoragePoolCmd.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ResourceUnavailableException; +import com.cloud.exception.InsufficientCapacityException; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.exception.ResourceAllocationException; +import com.cloud.exception.NetworkRuleConflictException; +import com.cloud.storage.StoragePool; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.context.CallContext; + +import java.util.logging.Logger; + +@APICommand(name = SyncStoragePoolCmd.APINAME, + description = "Sync storage pool with management server (currently supported for Datastore Cluster in VMware and syncs the datastores in it)", + responseObject = StoragePoolResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.15.1", + authorized = {RoleType.Admin} + ) +public class SyncStoragePoolCmd extends BaseAsyncCmd { + + public static final String APINAME = "syncStoragePool"; + public static final Logger LOGGER = Logger.getLogger(SyncStoragePoolCmd.class.getName()); + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, required = true, description = "Storage pool id") + private Long poolId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getPoolId() { + return poolId; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_SYNC_STORAGE_POOL; + } + + @Override + public String getEventDescription() { + return "Attempting to synchronise storage pool with management server"; + } + + @Override + public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException, NetworkRuleConflictException { + StoragePool result = _storageService.syncStoragePool(this); + if (result != null) { + StoragePoolResponse response = _responseGenerator.createStoragePoolResponse(result); + response.setResponseName("storagepool"); + this.setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to synchronise storage pool"); + } + } + + @Override + public String getCommandName() { + return APINAME.toLowerCase() + BaseAsyncCmd.RESPONSE_SUFFIX; + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } +} diff --git a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java index d86a1a619a9..b7cd5b8eb5e 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageProcessor.java @@ -30,6 +30,7 @@ import org.apache.cloudstack.storage.command.ForgetObjectCmd; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import com.cloud.agent.api.Answer; @@ -81,5 +82,7 @@ public interface StorageProcessor { Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd); - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd); + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd); + + public Answer syncVolumePath(SyncVolumePathCommand cmd); } diff --git a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java index 6c5b55a3af0..1a66034ce77 100644 --- a/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java +++ b/core/src/main/java/com/cloud/storage/resource/StorageSubsystemCommandHandlerBase.java @@ -34,6 +34,7 @@ import org.apache.cloudstack.storage.command.IntroduceObjectCmd; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import com.cloud.agent.api.Answer; import com.cloud.agent.api.Command; @@ -73,7 +74,9 @@ public class StorageSubsystemCommandHandlerBase implements StorageSubsystemComma } else if (command instanceof DirectDownloadCommand) { return processor.handleDownloadTemplateToPrimaryStorage((DirectDownloadCommand) command); } else if (command instanceof CheckDataStoreStoragePolicyComplainceCommand) { - return processor.CheckDataStoreStoragePolicyComplaince((CheckDataStoreStoragePolicyComplainceCommand) command); + return processor.checkDataStoreStoragePolicyCompliance((CheckDataStoreStoragePolicyComplainceCommand) command); + } else if (command instanceof SyncVolumePathCommand) { + return processor.syncVolumePath((SyncVolumePathCommand) command); } return new Answer((Command)command, false, "not implemented yet"); diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathAnswer.java new file mode 100644 index 00000000000..7560f7da59e --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathAnswer.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DiskTO; + +public class SyncVolumePathAnswer extends Answer { + private DiskTO disk; + + public SyncVolumePathAnswer() { + super(null); + } + + public SyncVolumePathAnswer(DiskTO disk) { + super(null); + setDisk(disk); + } + + public SyncVolumePathAnswer(String errMsg) { + super(null, false, errMsg); + } + + public DiskTO getDisk() { + return disk; + } + + public void setDisk(DiskTO disk) { + this.disk = disk; + } + +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathCommand.java new file mode 100644 index 00000000000..e655da4137b --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/SyncVolumePathCommand.java @@ -0,0 +1,49 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.storage.command; + +import com.cloud.agent.api.to.DiskTO; + +public class SyncVolumePathCommand extends StorageSubSystemCommand { + + private DiskTO disk; + + public SyncVolumePathCommand(final DiskTO disk) { + super(); + this.disk = disk; + } + + public DiskTO getDisk() { + return disk; + } + + public void setDisk(final DiskTO disk) { + this.disk = disk; + } + + @Override + public boolean executeInSequence() { + return false; + } + + @Override + public void setExecuteInSequence(boolean inSeq) { + + } +} diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index b20db8d3263..db41a2f099a 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -19,6 +19,7 @@ package com.cloud.storage; import java.math.BigDecimal; import java.util.List; +import com.cloud.agent.api.ModifyStoragePoolAnswer; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; import org.apache.cloudstack.framework.config.ConfigKey; @@ -272,4 +273,6 @@ public interface StorageManager extends StorageService { boolean isStoragePoolDatastoreClusterParent(StoragePool pool); + void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List childDatastoreAnswerList, long hostId); + } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java index 349baf05eb8..c27aeb0f652 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/StoragePoolHostDaoImpl.java @@ -23,7 +23,6 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; - import org.apache.log4j.Logger; import org.springframework.stereotype.Component; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 3375c6ff207..f56fdb85e58 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -130,4 +130,6 @@ public interface PrimaryDataStoreDao extends GenericDao { Integer countAll(); + List findPoolsByStorageType(String storageType); + } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index dfe1a6994d5..6b07ef95ca3 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -93,6 +93,7 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase AllFieldSearch.and("podId", AllFieldSearch.entity().getPodId(), Op.EQ); AllFieldSearch.and("clusterId", AllFieldSearch.entity().getClusterId(), Op.EQ); AllFieldSearch.and("storage_provider_name", AllFieldSearch.entity().getStorageProviderName(), Op.EQ); + AllFieldSearch.and("poolType", AllFieldSearch.entity().getPoolType(), Op.EQ); AllFieldSearch.done(); DcPodSearch = createSearchBuilder(); @@ -581,4 +582,11 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase sc.setParameters("status", StoragePoolStatus.Up); return listBy(sc); } + + @Override + public List findPoolsByStorageType(String storageType) { + SearchCriteria sc = AllFieldSearch.create(); + sc.setParameters("poolType", storageType); + return listBy(sc); + } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java index cb4ff74eed6..eb2262f0298 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/datastore/provider/DefaultHostListener.java @@ -22,16 +22,14 @@ import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.ModifyStoragePoolAnswer; import com.cloud.agent.api.ModifyStoragePoolCommand; -import com.cloud.agent.api.StoragePoolInfo; import com.cloud.alert.AlertManager; import com.cloud.exception.StorageConflictException; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage; import com.cloud.storage.StoragePool; import com.cloud.storage.StoragePoolHostVO; -import com.cloud.storage.StoragePoolStatus; import com.cloud.storage.dao.StoragePoolHostDao; -import com.cloud.storage.dao.StoragePoolTagsDao; +import com.cloud.storage.StorageManager; import com.cloud.utils.exception.CloudRuntimeException; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.HypervisorHostListener; @@ -43,9 +41,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import javax.inject.Inject; -import java.util.HashMap; import java.util.List; -import java.util.Map; public class DefaultHostListener implements HypervisorHostListener { private static final Logger s_logger = Logger.getLogger(DefaultHostListener.class); @@ -62,7 +58,7 @@ public class DefaultHostListener implements HypervisorHostListener { @Inject StoragePoolDetailsDao storagePoolDetailsDao; @Inject - StoragePoolTagsDao storagePoolTagsDao; + StorageManager storageManager; @Override public boolean hostAdded(long hostId) { @@ -104,43 +100,7 @@ public class DefaultHostListener implements HypervisorHostListener { updateStoragePoolHostVOAndDetails(poolVO, hostId, mspAnswer); if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { - for (ModifyStoragePoolAnswer childDataStoreAnswer : ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren()) { - StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); - StoragePoolVO dataStoreVO = primaryStoreDao.findPoolByUUID(childStoragePoolInfo.getUuid()); - if (dataStoreVO != null) { - continue; - } - dataStoreVO = new StoragePoolVO(); - dataStoreVO.setStorageProviderName(poolVO.getStorageProviderName()); - dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); - dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup); - dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); - dataStoreVO.setPort(poolVO.getPort()); - dataStoreVO.setName(childStoragePoolInfo.getName()); - dataStoreVO.setUuid(childStoragePoolInfo.getUuid()); - dataStoreVO.setDataCenterId(poolVO.getDataCenterId()); - dataStoreVO.setPodId(poolVO.getPodId()); - dataStoreVO.setClusterId(poolVO.getClusterId()); - dataStoreVO.setStatus(StoragePoolStatus.Up); - dataStoreVO.setUserInfo(poolVO.getUserInfo()); - dataStoreVO.setManaged(poolVO.isManaged()); - dataStoreVO.setCapacityIops(poolVO.getCapacityIops()); - dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes()); - dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); - dataStoreVO.setHypervisor(poolVO.getHypervisor()); - dataStoreVO.setScope(poolVO.getScope()); - dataStoreVO.setParent(poolVO.getId()); - - Map details = new HashMap<>(); - if(StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { - details.put("pool_type", childDataStoreAnswer.getPoolType()); - } - - List storageTags = storagePoolTagsDao.getStoragePoolTags(poolId); - primaryStoreDao.persist(dataStoreVO, details, storageTags); - - updateStoragePoolHostVOAndDetails(dataStoreVO, hostId, childDataStoreAnswer); - } + storageManager.syncDatastoreClusterStoragePool(poolId, ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren(), hostId); } s_logger.info("Connection established between storage pool " + pool + " and host " + hostId); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index c037f79cc72..34c610dd9a1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -59,6 +59,7 @@ import org.apache.cloudstack.storage.command.ResignatureAnswer; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -1922,8 +1923,14 @@ public class KVMStorageProcessor implements StorageProcessor { } @Override - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not currently applicable for KVMStorageProcessor"); return new Answer(cmd,false,"Not currently applicable for KVMStorageProcessor"); } + + @Override + public Answer syncVolumePath(SyncVolumePathCommand cmd) { + s_logger.info("SyncVolumePathCommand not currently applicable for KVMStorageProcessor"); + return new Answer(cmd, false, "Not currently applicable for KVMStorageProcessor"); + } } diff --git a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java index dd58bb573d9..f30df5d7444 100644 --- a/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java +++ b/plugins/hypervisors/ovm3/src/main/java/com/cloud/hypervisor/ovm3/resources/Ovm3StorageProcessor.java @@ -35,8 +35,9 @@ import org.apache.cloudstack.storage.command.ForgetObjectCmd; import org.apache.cloudstack.storage.command.IntroduceObjectCmd; import org.apache.cloudstack.storage.command.ResignatureAnswer; import org.apache.cloudstack.storage.command.ResignatureCommand; -import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; +import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -828,11 +829,17 @@ public class Ovm3StorageProcessor implements StorageProcessor { } @Override - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { LOGGER.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for Ovm3StorageProcessor"); return new Answer(cmd,false,"Not applicable used for Ovm3StorageProcessor"); } + @Override + public Answer syncVolumePath(SyncVolumePathCommand cmd) { + LOGGER.info("SyncVolumePathCommand not currently applicable for Ovm3StorageProcessor"); + return new Answer(cmd, false, "Not currently applicable for Ovm3StorageProcessor"); + } + @Override public Answer copyVolumeFromPrimaryToPrimary(CopyCommand cmd) { return null; diff --git a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java index 16579bc3886..8c8815cc08a 100644 --- a/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java +++ b/plugins/hypervisors/simulator/src/main/java/com/cloud/resource/SimulatorStorageProcessor.java @@ -41,6 +41,7 @@ import org.apache.cloudstack.storage.command.ResignatureAnswer; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; @@ -272,7 +273,12 @@ public class SimulatorStorageProcessor implements StorageProcessor { } @Override - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { + return new Answer(cmd, true, null); + } + + @Override + public Answer syncVolumePath(SyncVolumePathCommand cmd) { return new Answer(cmd, true, null); } } diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 2035cc736ad..81425f0a360 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -5037,10 +5037,12 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa String datacenterName = datastoreClusterPath.substring(0, pathstartPosition+1); String childPath = datacenterName + summary.getName(); poolInfo.setHostPath(childPath); - String uuid = UUID.nameUUIDFromBytes(((pool.getHost() + childPath)).getBytes()).toString(); + String uuid = childDsMo.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); + if (uuid == null) { + uuid = UUID.nameUUIDFromBytes(((pool.getHost() + childPath)).getBytes()).toString(); + } poolInfo.setUuid(uuid); poolInfo.setLocalPath(cmd.LOCAL_PATH_PREFIX + File.separator + uuid); - answer.setPoolInfo(poolInfo); answer.setPoolType(summary.getType()); answer.setLocalDatastoreName(morDatastore.getValue()); diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java index b06c1bef69c..eb6251a2a39 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/storage/resource/VmwareStorageProcessor.java @@ -50,6 +50,8 @@ import org.apache.cloudstack.storage.command.ResignatureAnswer; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -3892,7 +3894,7 @@ public class VmwareStorageProcessor implements StorageProcessor { } @Override - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { String primaryStorageNameLabel = cmd.getStoragePool().getUuid(); String storagePolicyId = cmd.getStoragePolicyId(); VmwareContext context = hostService.getServiceContext(cmd); @@ -3962,4 +3964,81 @@ public class VmwareStorageProcessor implements StorageProcessor { throw new CloudRuntimeException(msg, e); } } + + @Override + public Answer syncVolumePath(SyncVolumePathCommand cmd) { + DiskTO disk = cmd.getDisk(); + VolumeObjectTO volumeTO = (VolumeObjectTO)disk.getData(); + DataStoreTO primaryStore = volumeTO.getDataStore(); + String volumePath = volumeTO.getPath(); + String vmName = volumeTO.getVmName(); + + boolean datastoreChangeObserved = false; + boolean volumePathChangeObserved = false; + String chainInfo = null; + try { + VmwareContext context = hostService.getServiceContext(null); + VmwareHypervisorHost hyperHost = hostService.getHyperHost(context, null); + VirtualMachineMO vmMo = hyperHost.findVmOnHyperHost(vmName); + if (vmMo == null) { + vmMo = hyperHost.findVmOnPeerHyperHost(vmName); + if (vmMo == null) { + String msg = "Unable to find the VM to execute SyncVolumePathCommand, vmName: " + vmName; + s_logger.error(msg); + throw new Exception(msg); + } + } + + String datastoreUUID = primaryStore.getUuid(); + if (disk.getDetails().get(DiskTO.PROTOCOL_TYPE) != null && disk.getDetails().get(DiskTO.PROTOCOL_TYPE).equalsIgnoreCase("DatastoreCluster")) { + VirtualMachineDiskInfo matchingExistingDisk = getMatchingExistingDisk(hyperHost, context, vmMo, disk); + VirtualMachineDiskInfoBuilder diskInfoBuilder = vmMo.getDiskInfoBuilder(); + if (diskInfoBuilder != null && matchingExistingDisk != null) { + String[] diskChain = matchingExistingDisk.getDiskChain(); + assert (diskChain.length > 0); + DatastoreFile file = new DatastoreFile(diskChain[0]); + if (!file.getFileBaseName().equalsIgnoreCase(volumePath)) { + if (s_logger.isInfoEnabled()) + s_logger.info("Detected disk-chain top file change on volume: " + volumeTO.getId() + " " + volumePath + " -> " + file.getFileBaseName()); + volumePathChangeObserved = true; + volumePath = file.getFileBaseName(); + volumeTO.setPath(volumePath); + chainInfo = _gson.toJson(matchingExistingDisk); + } + + DatastoreMO diskDatastoreMofromVM = getDiskDatastoreMofromVM(hyperHost, context, vmMo, disk, diskInfoBuilder); + if (diskDatastoreMofromVM != null) { + String actualPoolUuid = diskDatastoreMofromVM.getCustomFieldValue(CustomFieldConstants.CLOUD_UUID); + if (!actualPoolUuid.equalsIgnoreCase(primaryStore.getUuid())) { + s_logger.warn(String.format("Volume %s found to be in a different storage pool %s", volumePath, actualPoolUuid)); + datastoreChangeObserved = true; + datastoreUUID = actualPoolUuid; + chainInfo = _gson.toJson(matchingExistingDisk); + } + } + } + } + + SyncVolumePathAnswer answer = new SyncVolumePathAnswer(disk); + if (datastoreChangeObserved) { + answer.setContextParam("datastoreName", datastoreUUID); + } + if (volumePathChangeObserved) { + answer.setContextParam("volumePath", volumePath); + } + if (chainInfo != null && !chainInfo.isEmpty()) { + answer.setContextParam("chainInfo", chainInfo); + } + + return answer; + } catch (Throwable e) { + if (e instanceof RemoteException) { + s_logger.warn("Encounter remote exception to vCenter, invalidate VMware session context"); + + hostService.invalidateServiceContext(null); + } + + return new SyncVolumePathAnswer("Failed to process SyncVolumePathCommand due to " + e.getMessage()); + } + } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java index b3fce78af86..a035eac30fb 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java @@ -50,6 +50,7 @@ import org.apache.cloudstack.storage.command.ResignatureAnswer; import org.apache.cloudstack.storage.command.ResignatureCommand; import org.apache.cloudstack.storage.command.SnapshotAndCopyAnswer; import org.apache.cloudstack.storage.command.SnapshotAndCopyCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -217,11 +218,17 @@ public class XenServerStorageProcessor implements StorageProcessor { } @Override - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); } + @Override + public Answer syncVolumePath(SyncVolumePathCommand cmd) { + s_logger.info("SyncVolumePathCommand not currently applicable for XenServerStorageProcessor"); + return new Answer(cmd, false, "Not currently applicable for XenServerStorageProcessor"); + } + @Override public AttachAnswer attachIso(final AttachCommand cmd) { final DiskTO disk = cmd.getDisk(); diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java index 97a4efa06d7..7e2c701e51a 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java @@ -30,6 +30,7 @@ import java.util.UUID; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.CopyCmdAnswer; import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -915,11 +916,17 @@ public class Xenserver625StorageProcessor extends XenServerStorageProcessor { } @Override - public Answer CheckDataStoreStoragePolicyComplaince(CheckDataStoreStoragePolicyComplainceCommand cmd) { + public Answer checkDataStoreStoragePolicyCompliance(CheckDataStoreStoragePolicyComplainceCommand cmd) { s_logger.info("'CheckDataStoreStoragePolicyComplainceCommand' not applicable used for XenServerStorageProcessor"); return new Answer(cmd,false,"Not applicable used for XenServerStorageProcessor"); } + @Override + public Answer syncVolumePath(SyncVolumePathCommand cmd) { + s_logger.info("SyncVolumePathCommand not currently applicable for XenServerStorageProcessor"); + return new Answer(cmd, false, "Not currently applicable for XenServerStorageProcessor"); + } + @Override public Answer copyVolumeFromPrimaryToSecondary(final CopyCommand cmd) { final Connection conn = hypervisorResource.getConnection(); diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 8a1834f7e16..e68f03d2ed6 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -207,6 +207,7 @@ import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForM import org.apache.cloudstack.api.command.admin.storage.UpdateCloudToUseObjectStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.swift.AddSwiftCmd; import org.apache.cloudstack.api.command.admin.swift.ListSwiftsCmd; import org.apache.cloudstack.api.command.admin.systemvm.DestroySystemVmCmd; @@ -3039,6 +3040,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(FindStoragePoolsForMigrationCmd.class); cmdList.add(PreparePrimaryStorageForMaintenanceCmd.class); cmdList.add(UpdateStoragePoolCmd.class); + cmdList.add(SyncStoragePoolCmd.class); cmdList.add(UpdateImageStoreCmd.class); cmdList.add(DestroySystemVmCmd.class); cmdList.add(ListSystemVMsCmd.class); diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 62119ff1edd..5e7be308323 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -32,6 +32,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; +import java.util.LinkedHashSet; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -52,6 +54,7 @@ import org.apache.cloudstack.api.command.admin.storage.DeleteImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.DeletePoolCmd; import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd; +import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.engine.subsystem.api.storage.ClusterScope; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; @@ -90,6 +93,8 @@ import org.apache.cloudstack.management.ManagementServerHost; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.storage.command.CheckDataStoreStoragePolicyComplainceCommand; import org.apache.cloudstack.storage.command.DettachCommand; +import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; +import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; @@ -104,6 +109,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -119,6 +125,8 @@ import com.cloud.agent.api.StoragePoolInfo; import com.cloud.agent.api.VolumeStatsEntry; import com.cloud.agent.api.to.DataTO; import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.ModifyStoragePoolCommand; +import com.cloud.agent.api.ModifyStoragePoolAnswer; import com.cloud.agent.manager.Commands; import com.cloud.api.ApiDBUtils; import com.cloud.api.query.dao.TemplateJoinDao; @@ -1624,14 +1632,14 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C if (primaryStorage.getStatus() == StoragePoolStatus.PrepareForMaintenance) { throw new CloudRuntimeException(String.format("There is already a job running for preparation for maintenance of the storage pool %s", primaryStorage.getUuid())); } - handlePrepareDatastoreCluserMaintenance(lifeCycle, primaryStorageId); + handlePrepareDatastoreClusterMaintenance(lifeCycle, primaryStorageId); } lifeCycle.maintain(store); return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); } - private void handlePrepareDatastoreCluserMaintenance(DataStoreLifeCycle lifeCycle, Long primaryStorageId) { + private void handlePrepareDatastoreClusterMaintenance(DataStoreLifeCycle lifeCycle, Long primaryStorageId) { StoragePoolVO datastoreCluster = _storagePoolDao.findById(primaryStorageId); datastoreCluster.setStatus(StoragePoolStatus.PrepareForMaintenance); _storagePoolDao.update(datastoreCluster.getId(), datastoreCluster); @@ -1705,6 +1713,261 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C return (PrimaryDataStoreInfo)_dataStoreMgr.getDataStore(primaryStorage.getId(), DataStoreRole.Primary); } + @Override + @ActionEvent(eventType = EventTypes.EVENT_SYNC_STORAGE_POOL, eventDescription = "synchronising storage pool with management server", async = true) + public StoragePool syncStoragePool(SyncStoragePoolCmd cmd) { + Long poolId = cmd.getPoolId(); + StoragePoolVO pool = _storagePoolDao.findById(poolId); + + if (pool == null) { + String msg = String.format("Unable to obtain lock on the storage pool record while syncing storage pool [%s] with management server", pool.getUuid()); + s_logger.error(msg); + throw new InvalidParameterValueException(msg); + } + + if (!pool.getPoolType().equals(StoragePoolType.DatastoreCluster)) { + throw new InvalidParameterValueException("SyncStoragePool API is currently supported only for storage type of datastore cluster"); + } + + if (!pool.getStatus().equals(StoragePoolStatus.Up)) { + throw new InvalidParameterValueException(String.format("Primary storage with id %s is not ready for syncing, as the status is %s", pool.getUuid(), pool.getStatus().toString())); + } + + // find the host + List poolIds = new ArrayList<>(); + poolIds.add(poolId); + List hosts = _storagePoolHostDao.findHostsConnectedToPools(poolIds); + if (hosts.size() > 0) { + Long hostId = hosts.get(0); + ModifyStoragePoolCommand modifyStoragePoolCommand = new ModifyStoragePoolCommand(true, pool); + final Answer answer = _agentMgr.easySend(hostId, modifyStoragePoolCommand); + + if (answer == null) { + throw new CloudRuntimeException(String.format("Unable to get an answer to the modify storage pool command %s", pool.getUuid())); + } + + if (!answer.getResult()) { + throw new CloudRuntimeException(String.format("Unable to process ModifyStoragePoolCommand for pool %s on the host %s due to ", pool.getUuid(), hostId, answer.getDetails())); + } + + assert (answer instanceof ModifyStoragePoolAnswer) : "Well, now why won't you actually return the ModifyStoragePoolAnswer when it's ModifyStoragePoolCommand? Pool=" + + pool.getId() + "Host=" + hostId; + ModifyStoragePoolAnswer mspAnswer = (ModifyStoragePoolAnswer) answer; + StoragePoolVO poolVO = _storagePoolDao.findById(poolId); + updateStoragePoolHostVOAndBytes(poolVO, hostId, mspAnswer); + validateChildDatastoresToBeAddedInUpState(poolVO, mspAnswer.getDatastoreClusterChildren()); + syncDatastoreClusterStoragePool(poolId, mspAnswer.getDatastoreClusterChildren(), hostId); + for (ModifyStoragePoolAnswer childDataStoreAnswer : mspAnswer.getDatastoreClusterChildren()) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childStoragePoolInfo.getUuid()); + for (Long host : hosts) { + updateStoragePoolHostVOAndBytes(dataStoreVO, host, childDataStoreAnswer); + } + } + + } else { + throw new CloudRuntimeException(String.format("Unable to sync storage pool [%s] as there no connected hosts to the storage pool", pool.getUuid())); + } + return (PrimaryDataStoreInfo) _dataStoreMgr.getDataStore(pool.getId(), DataStoreRole.Primary); + } + + public void syncDatastoreClusterStoragePool(long datastoreClusterPoolId, List childDatastoreAnswerList, long hostId) { + StoragePoolVO datastoreClusterPool = _storagePoolDao.findById(datastoreClusterPoolId); + List storageTags = _storagePoolTagsDao.getStoragePoolTags(datastoreClusterPoolId); + List childDatastores = _storagePoolDao.listChildStoragePoolsInDatastoreCluster(datastoreClusterPoolId); + Set childDatastoreUUIDs = new HashSet<>(); + for (StoragePoolVO childDatastore : childDatastores) { + childDatastoreUUIDs.add(childDatastore.getUuid()); + } + + for (ModifyStoragePoolAnswer childDataStoreAnswer : childDatastoreAnswerList) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childStoragePoolInfo.getUuid()); + if (dataStoreVO == null && childDataStoreAnswer.getPoolType().equalsIgnoreCase("NFS")) { + List nfsStoragePools = _storagePoolDao.findPoolsByStorageType(StoragePoolType.NetworkFilesystem.toString()); + for (StoragePoolVO storagePool : nfsStoragePools) { + String storagePoolUUID = storagePool.getUuid(); + if (childStoragePoolInfo.getName().equalsIgnoreCase(storagePoolUUID.replaceAll("-", ""))) { + dataStoreVO = storagePool; + break; + } + } + } + if (dataStoreVO != null) { + if (dataStoreVO.getParent() != datastoreClusterPoolId) { + s_logger.debug(String.format("Storage pool %s with uuid %s is found to be under datastore cluster %s at vCenter, " + + "so moving the storage pool to be a child storage pool under the datastore cluster in CloudStack management server", + childStoragePoolInfo.getName(), childStoragePoolInfo.getUuid(), datastoreClusterPool.getName())); + dataStoreVO.setParent(datastoreClusterPoolId); + _storagePoolDao.update(dataStoreVO.getId(), dataStoreVO); + if (CollectionUtils.isNotEmpty(storageTags)) { + storageTags.addAll(_storagePoolTagsDao.getStoragePoolTags(dataStoreVO.getId())); + } else { + storageTags = _storagePoolTagsDao.getStoragePoolTags(dataStoreVO.getId()); + } + if (CollectionUtils.isNotEmpty(storageTags)) { + Set set = new LinkedHashSet<>(storageTags); + storageTags.clear(); + storageTags.addAll(set); + if (s_logger.isDebugEnabled()) { + s_logger.debug("Updating Storage Pool Tags to :" + storageTags); + } + _storagePoolTagsDao.persist(dataStoreVO.getId(), storageTags); + } + } else { + // This is to find datastores which are removed from datastore cluster. + // The final set childDatastoreUUIDs contains the UUIDs of child datastores which needs to be removed from datastore cluster + childDatastoreUUIDs.remove(childStoragePoolInfo.getUuid()); + } + } else { + dataStoreVO = createChildDatastoreVO(datastoreClusterPool, childDataStoreAnswer); + } + updateStoragePoolHostVOAndBytes(dataStoreVO, hostId, childDataStoreAnswer); + } + + handleRemoveChildStoragePoolFromDatastoreCluster(childDatastoreUUIDs); + } + + private void validateChildDatastoresToBeAddedInUpState(StoragePoolVO datastoreClusterPool, List childDatastoreAnswerList) { + for (ModifyStoragePoolAnswer childDataStoreAnswer : childDatastoreAnswerList) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childStoragePoolInfo.getUuid()); + if (dataStoreVO == null && childDataStoreAnswer.getPoolType().equalsIgnoreCase("NFS")) { + List nfsStoragePools = _storagePoolDao.findPoolsByStorageType(StoragePoolType.NetworkFilesystem.toString()); + for (StoragePoolVO storagePool : nfsStoragePools) { + String storagePoolUUID = storagePool.getUuid(); + if (childStoragePoolInfo.getName().equalsIgnoreCase(storagePoolUUID.replaceAll("-", ""))) { + dataStoreVO = storagePool; + break; + } + } + } + if (dataStoreVO != null && !dataStoreVO.getStatus().equals(StoragePoolStatus.Up)) { + String msg = String.format("Cannot synchronise datastore cluster %s because primary storage with id %s is not ready for syncing, " + + "as the status is %s", datastoreClusterPool.getUuid(), dataStoreVO.getUuid(), dataStoreVO.getStatus().toString()); + throw new CloudRuntimeException(msg); + } + } + } + + private StoragePoolVO createChildDatastoreVO(StoragePoolVO datastoreClusterPool, ModifyStoragePoolAnswer childDataStoreAnswer) { + StoragePoolInfo childStoragePoolInfo = childDataStoreAnswer.getPoolInfo(); + List storageTags = _storagePoolTagsDao.getStoragePoolTags(datastoreClusterPool.getId()); + + StoragePoolVO dataStoreVO = new StoragePoolVO(); + dataStoreVO.setStorageProviderName(datastoreClusterPool.getStorageProviderName()); + dataStoreVO.setHostAddress(childStoragePoolInfo.getHost()); + dataStoreVO.setPoolType(Storage.StoragePoolType.PreSetup); + dataStoreVO.setPath(childStoragePoolInfo.getHostPath()); + dataStoreVO.setPort(datastoreClusterPool.getPort()); + dataStoreVO.setName(childStoragePoolInfo.getName()); + dataStoreVO.setUuid(childStoragePoolInfo.getUuid()); + dataStoreVO.setDataCenterId(datastoreClusterPool.getDataCenterId()); + dataStoreVO.setPodId(datastoreClusterPool.getPodId()); + dataStoreVO.setClusterId(datastoreClusterPool.getClusterId()); + dataStoreVO.setStatus(StoragePoolStatus.Up); + dataStoreVO.setUserInfo(datastoreClusterPool.getUserInfo()); + dataStoreVO.setManaged(datastoreClusterPool.isManaged()); + dataStoreVO.setCapacityIops(datastoreClusterPool.getCapacityIops()); + dataStoreVO.setCapacityBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes()); + dataStoreVO.setUsedBytes(childDataStoreAnswer.getPoolInfo().getCapacityBytes() - childDataStoreAnswer.getPoolInfo().getAvailableBytes()); + dataStoreVO.setHypervisor(datastoreClusterPool.getHypervisor()); + dataStoreVO.setScope(datastoreClusterPool.getScope()); + dataStoreVO.setParent(datastoreClusterPool.getId()); + + Map details = new HashMap<>(); + if(org.apache.commons.lang.StringUtils.isNotEmpty(childDataStoreAnswer.getPoolType())) { + details.put("pool_type", childDataStoreAnswer.getPoolType()); + } + _storagePoolDao.persist(dataStoreVO, details, storageTags); + return dataStoreVO; + } + + private void handleRemoveChildStoragePoolFromDatastoreCluster(Set childDatastoreUUIDs) { + + for (String childDatastoreUUID : childDatastoreUUIDs) { + StoragePoolVO dataStoreVO = _storagePoolDao.findPoolByUUID(childDatastoreUUID); + List allVolumes = _volumeDao.findByPoolId(dataStoreVO.getId()); + allVolumes.removeIf(volumeVO -> volumeVO.getInstanceId() == null); + allVolumes.removeIf(volumeVO -> volumeVO.getState() != Volume.State.Ready); + for (VolumeVO volume : allVolumes) { + VMInstanceVO vmInstance = _vmInstanceDao.findById(volume.getInstanceId()); + if (vmInstance == null) { + continue; + } + long volumeId = volume.getId(); + Long hostId = vmInstance.getHostId(); + if (hostId == null) { + hostId = vmInstance.getLastHostId(); + } + HostVO hostVO = _hostDao.findById(hostId); + + // Prepare for the syncvolumepath command + DataTO volTO = volFactory.getVolume(volume.getId()).getTO(); + DiskTO disk = new DiskTO(volTO, volume.getDeviceId(), volume.getPath(), volume.getVolumeType()); + Map details = new HashMap(); + details.put(DiskTO.PROTOCOL_TYPE, Storage.StoragePoolType.DatastoreCluster.toString()); + disk.setDetails(details); + + s_logger.debug(String.format("Attempting to process SyncVolumePathCommand for the volume %d on the host %d with state %s", volumeId, hostId, hostVO.getResourceState())); + SyncVolumePathCommand cmd = new SyncVolumePathCommand(disk); + final Answer answer = _agentMgr.easySend(hostId, cmd); + // validate answer + if (answer == null) { + throw new CloudRuntimeException("Unable to get an answer to the SyncVolumePath command for volume " + volumeId); + } + if (!answer.getResult()) { + throw new CloudRuntimeException("Unable to process SyncVolumePathCommand for the volume" + volumeId + " to the host " + hostId + " due to " + answer.getDetails()); + } + assert (answer instanceof SyncVolumePathAnswer) : "Well, now why won't you actually return the SyncVolumePathAnswer when it's SyncVolumePathCommand? volume=" + + volume.getUuid() + "Host=" + hostId; + + // check for the changed details of volume and update database + VolumeVO volumeVO = _volumeDao.findById(volumeId); + String datastoreName = answer.getContextParam("datastoreName"); + if (datastoreName != null) { + StoragePoolVO storagePoolVO = _storagePoolDao.findByUuid(datastoreName); + if (storagePoolVO != null) { + volumeVO.setPoolId(storagePoolVO.getId()); + } else { + s_logger.warn(String.format("Unable to find datastore %s while updating the new datastore of the volume %d", datastoreName, volumeId)); + } + } + + String volumePath = answer.getContextParam("volumePath"); + if (volumePath != null) { + volumeVO.setPath(volumePath); + } + + String chainInfo = answer.getContextParam("chainInfo"); + if (chainInfo != null) { + volumeVO.setChainInfo(chainInfo); + } + + _volumeDao.update(volumeVO.getId(), volumeVO); + } + dataStoreVO.setParent(0L); + _storagePoolDao.update(dataStoreVO.getId(), dataStoreVO); + } + + } + + private void updateStoragePoolHostVOAndBytes(StoragePool pool, long hostId, ModifyStoragePoolAnswer mspAnswer) { + StoragePoolHostVO poolHost = _storagePoolHostDao.findByPoolHost(pool.getId(), hostId); + if (poolHost == null) { + poolHost = new StoragePoolHostVO(pool.getId(), hostId, mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); + _storagePoolHostDao.persist(poolHost); + } else { + poolHost.setLocalPath(mspAnswer.getPoolInfo().getLocalPath().replaceAll("//", "/")); + } + + StoragePoolVO poolVO = _storagePoolDao.findById(pool.getId()); + poolVO.setUsedBytes(mspAnswer.getPoolInfo().getCapacityBytes() - mspAnswer.getPoolInfo().getAvailableBytes()); + poolVO.setCapacityBytes(mspAnswer.getPoolInfo().getCapacityBytes()); + + _storagePoolDao.update(pool.getId(), poolVO); + } + protected class StorageGarbageCollector extends ManagedContextRunnable { public StorageGarbageCollector() { diff --git a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java index 3e8822e67eb..772b947128d 100644 --- a/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java +++ b/server/src/main/java/com/cloud/storage/StoragePoolAutomationImpl.java @@ -35,6 +35,7 @@ import org.springframework.stereotype.Component; import com.cloud.agent.AgentManager; import com.cloud.agent.api.Answer; import com.cloud.agent.api.ModifyStoragePoolCommand; +import com.cloud.agent.api.ModifyStoragePoolAnswer; import com.cloud.alert.AlertManager; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -100,6 +101,8 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { ManagementServer server; @Inject DataStoreProviderManager providerMgr; + @Inject + StorageManager storageManager; @Override public boolean maintain(DataStore store) { @@ -162,6 +165,10 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { if (s_logger.isDebugEnabled()) { s_logger.debug("ModifyStoragePool false succeeded"); } + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + s_logger.debug(String.format("Started synchronising datastore cluster storage pool %s with vCenter", pool.getUuid())); + storageManager.syncDatastoreClusterStoragePool(pool.getId(), ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren(), host.getId()); + } } } // check to see if other ps exist @@ -323,6 +330,10 @@ public class StoragePoolAutomationImpl implements StoragePoolAutomation { if (s_logger.isDebugEnabled()) { s_logger.debug("ModifyStoragePool add succeeded"); } + if (pool.getPoolType() == Storage.StoragePoolType.DatastoreCluster) { + s_logger.debug(String.format("Started synchronising datastore cluster storage pool %s with vCenter", pool.getUuid())); + storageManager.syncDatastoreClusterStoragePool(pool.getId(), ((ModifyStoragePoolAnswer) answer).getDatastoreClusterChildren(), host.getId()); + } } } diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java index 3dd926470a0..af081e539e1 100644 --- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java +++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java @@ -2265,7 +2265,7 @@ public class UserVmManagerImpl extends ManagerBase implements UserVmManager, Vir VMInstanceVO vmInstance = _vmInstanceDao.findById(vmId); // only load running vms. For stopped vms get loaded on starting - if (vmInstance.getState() == State.Running) { + if (vmInstance != null && vmInstance.getState() == State.Running) { VmAndCountDetails vmAndCount = new VmAndCountDetails(vmId, VmIpFetchTrialMax.value()); vmIdCountMap.put(nicId, vmAndCount); } diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index 84140eefadb..9107b7cba84 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -95,6 +95,7 @@ known_categories = { 'StorageMaintenance': 'Storage Pool', 'StoragePool': 'Storage Pool', 'StorageProvider': 'Storage Pool', + 'syncStoragePool': 'Storage Pool', 'SecurityGroup': 'Security Group', 'SSH': 'SSH', 'register': 'Registration', diff --git a/ui/src/components/view/DetailSettings.vue b/ui/src/components/view/DetailSettings.vue index 85360b8d642..77aac15e927 100644 --- a/ui/src/components/view/DetailSettings.vue +++ b/ui/src/components/view/DetailSettings.vue @@ -52,18 +52,8 @@ :dataSource="detailOptions[newKey]" :placeholder="$t('label.value')" @change="e => onAddInputChange(e, 'newValue')" /> - - - - - - - - + +

{{ $t(error) }}

@@ -90,14 +80,10 @@ slot="actions" v-if="!disableSettings && 'updateTemplate' in $store.getters.apis && 'updateVirtualMachine' in $store.getters.apis && isAdminOrOwner() && allowEditOfDetail(item.name)"> - - - - - - - + + @@ -113,7 +99,7 @@ :cancelText="$t('label.no')" placement="left" > - + @@ -123,8 +109,10 @@ + + diff --git a/ui/src/views/compute/InstanceTab.vue b/ui/src/views/compute/InstanceTab.vue index 8ad259b290a..1a169d756c7 100644 --- a/ui/src/views/compute/InstanceTab.vue +++ b/ui/src/views/compute/InstanceTab.vue @@ -67,61 +67,47 @@ - - - - - - - - - - - - - - - - - - - - + + + + + + + + @@ -251,9 +237,10 @@ :okText="$t('label.yes')" :cancelText="$t('label.no')" > - {{ ip.ipaddress }} @@ -274,6 +261,7 @@ import DetailsTab from '@/components/view/DetailsTab' import DetailSettings from '@/components/view/DetailSettings' import NicsTable from '@/views/network/NicsTable' import ListResourceTable from '@/components/view/ListResourceTable' +import TooltipButton from '@/components/view/TooltipButton' export default { name: 'InstanceTab', @@ -283,7 +271,8 @@ export default { DetailSettings, NicsTable, Status, - ListResourceTable + ListResourceTable, + TooltipButton }, mixins: [mixinDevice], props: { diff --git a/ui/src/views/compute/backup/BackupSchedule.vue b/ui/src/views/compute/backup/BackupSchedule.vue index daf31cc31dc..0c31e86c26c 100644 --- a/ui/src/views/compute/backup/BackupSchedule.vue +++ b/ui/src/views/compute/backup/BackupSchedule.vue @@ -58,18 +58,14 @@ @@ -78,9 +74,13 @@