diff --git a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml index 17c5002c718..49c668f50e8 100644 --- a/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml +++ b/engine/orchestration/src/main/resources/META-INF/cloudstack/core/spring-engine-orchestration-core-context.xml @@ -88,6 +88,7 @@ + diff --git a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java index c859e888ac0..7ea30035a52 100644 --- a/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/backup/KVMBackupExportServiceImpl.java @@ -17,6 +17,9 @@ package org.apache.cloudstack.backup; +import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; +import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; + import java.time.Instant; import java.util.ArrayList; import java.util.Collections; @@ -45,6 +48,10 @@ import org.apache.cloudstack.backup.dao.BackupDao; import org.apache.cloudstack.backup.dao.ImageTransferDao; import org.apache.cloudstack.context.CallContext; import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; +import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; @@ -71,23 +78,31 @@ import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.StoragePoolHostDao; import com.cloud.storage.dao.VolumeDao; import com.cloud.storage.dao.VolumeDetailsDao; +import com.cloud.user.Account; import com.cloud.user.AccountService; import com.cloud.user.User; import com.cloud.utils.NumbersUtil; +import com.cloud.utils.Pair; +import com.cloud.utils.ReflectionUse; import com.cloud.utils.component.ManagerBase; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachine.State; import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.VmWork; +import com.cloud.vm.VmWorkConstants; +import com.cloud.vm.VmWorkJobHandler; +import com.cloud.vm.VmWorkJobHandlerProxy; +import com.cloud.vm.VmWorkSerializer; import com.cloud.vm.dao.VMInstanceDao; import com.cloud.vm.dao.VMInstanceDetailsDao; -import static org.apache.cloudstack.backup.BackupManager.BackupFrameworkEnabled; -import static org.apache.cloudstack.backup.BackupManager.BackupProviderPlugin; - @Component -public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackupExportService { +public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackupExportService, VmWorkJobHandler { + public static final String VM_WORK_JOB_HANDLER = KVMBackupExportServiceImpl.class.getSimpleName(); + private static final long BACKUP_FINALIZE_WAIT_CHECK_INTERVAL = 15 * 1000L; @Inject private VMInstanceDao vmInstanceDao; @@ -122,8 +137,13 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup @Inject AccountService accountService; + @Inject + AsyncJobManager asyncJobManager; + private Timer imageTransferTimer; + VmWorkJobHandlerProxy jobHandlerProxy = new VmWorkJobHandlerProxy(this); + private boolean isKVMBackupExportServiceSupported(Long zoneId) { return !BackupFrameworkEnabled.value() || StringUtils.equals("dummy", BackupProviderPlugin.valueIn(zoneId)); } @@ -189,6 +209,28 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup backupDao.remove(backup.getId()); } + protected void queueBackupFinalizeWaitWorkJob(final VMInstanceVO vm, final BackupVO backup) { + final CallContext context = CallContext.current(); + final Account callingAccount = context.getCallingAccount(); + final long callingUserId = context.getCallingUserId(); + + VmWorkJobVO workJob = new VmWorkJobVO(context.getContextId()); + workJob.setDispatcher(VmWorkConstants.VM_WORK_JOB_DISPATCHER); + workJob.setCmd(VmWorkWaitForBackupFinalize.class.getName()); + workJob.setAccountId(callingAccount.getId()); + workJob.setUserId(callingUserId); + workJob.setStep(VmWorkJobVO.Step.Starting); + workJob.setVmType(VirtualMachine.Type.User); + workJob.setVmInstanceId(vm.getId()); + workJob.setRelated(AsyncJobExecutionContext.getOriginJobId()); + + VmWorkWaitForBackupFinalize workInfo = new VmWorkWaitForBackupFinalize( + callingUserId, callingAccount.getId(), vm.getId(), VM_WORK_JOB_HANDLER, backup.getId()); + workJob.setCmdInfo(VmWorkSerializer.serialize(workInfo)); + + asyncJobManager.submitAsyncJob(workJob, VmWorkConstants.VM_WORK_QUEUE, vm.getId()); + } + @Override public Backup startBackup(StartBackupCmd cmd) { BackupVO backup = backupDao.findById(cmd.getEntityId()); @@ -246,6 +288,7 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup // Update backup with checkpoint creation time backup.setCheckpointCreateTime(answer.getCheckpointCreateTime()); updateBackupState(backup, Backup.Status.ReadyForTransfer); + queueBackupFinalizeWaitWorkJob(vm, backup); return backup; } @@ -873,6 +916,43 @@ public class KVMBackupExportServiceImpl extends ManagerBase implements KVMBackup return true; } + @ReflectionUse + public Pair orchestrateWaitForBackupFinalize(VmWorkWaitForBackupFinalize work) { + return waitForBackupTerminalState(work.getBackupId()); + } + + @Override + public Pair handleVmWorkJob(VmWork work) throws Exception { + return jobHandlerProxy.handleVmWorkJob(work); + } + + protected Pair waitForBackupTerminalState(final long backupId) { + while (true) { + final BackupVO backup = backupDao.findByIdIncludingRemoved(backupId); + if (backup == null) { + RuntimeException ex = new CloudRuntimeException(String.format("Backup %d not found while waiting for finalize", backupId)); + return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); + } + + if (backup.getStatus() == Backup.Status.BackedUp) { + return new Pair<>(JobInfo.Status.SUCCEEDED, asyncJobManager.marshallResultObject(backup.getId())); + } + + if (backup.getStatus() == Backup.Status.Failed || backup.getStatus() == Backup.Status.Error) { + RuntimeException ex = new CloudRuntimeException(String.format("Backup %d reached terminal failure state: %s", backupId, backup.getStatus())); + return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); + } + + try { + Thread.sleep(BACKUP_FINALIZE_WAIT_CHECK_INTERVAL); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + RuntimeException ex = new CloudRuntimeException(String.format("Interrupted while waiting for backup %d finalize", backupId), e); + return new Pair<>(JobInfo.Status.FAILED, asyncJobManager.marshallResultObject(ex)); + } + } + } + private void pollImageTransferProgress() { try { List transferringTransfers = imageTransferDao.listByPhaseAndDirection( diff --git a/server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java b/server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java new file mode 100644 index 00000000000..ac64b47aa3e --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/backup/VmWorkWaitForBackupFinalize.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.backup; + +import com.cloud.vm.VmWork; + +public class VmWorkWaitForBackupFinalize extends VmWork { + private static final long serialVersionUID = 2209426364298601717L; + + private final long backupId; + + public VmWorkWaitForBackupFinalize(long userId, long accountId, long vmId, String handlerName, long backupId) { + super(userId, accountId, vmId, handlerName); + this.backupId = backupId; + } + + public long getBackupId() { + return backupId; + } +}