etan39
/
cloudstack
mirror of https://github.com/apache/cloudstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CleanUp Async Jobs after mgmt server maintenance (#8394) 

This PR fixes moves resources stuck in transition state during async job cleanup

Problem:
During maintenance of the management server, other servers in the cluster or the same server after a restart initiate async job cleanup. However, this process leaves resources in a transitional state. The only recovery option currently available is to make direct database changes.

Solution:
This PR introduces a resolution by changing Volume, Virtual Machine, and Network resources from their transitional states. This adjustment enables the reattempt of failed operations without the need for manual database modifications.
This commit is contained in:
kishankavala 2024-01-19 13:26:25 +05:30 committed by GitHub
parent 8d42ca8ccf
commit 80bbb29abf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 274 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
api/src/main/java/com/cloud/storage/Volume.java
View File

@ -39,30 +39,38 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba
};
enum State {
Allocated("The volume is allocated but has not been created yet."),
Creating("The volume is being created. getPoolId() should reflect the pool where it is being created."),
Ready("The volume is ready to be used."),
Migrating("The volume is migrating to other storage pool"),
Snapshotting("There is a snapshot created on this volume, not backed up to secondary storage yet"),
RevertSnapshotting("There is a snapshot created on this volume, the volume is being reverting from snapshot"),
Resizing("The volume is being resized"),
Expunging("The volume is being expunging"),
Expunged("The volume has been expunged, and can no longer be recovered"),
Destroy("The volume is destroyed, and can be recovered."),
Destroying("The volume is destroying, and can't be recovered."),
UploadOp("The volume upload operation is in progress or in short the volume is on secondary storage"),
Copying("Volume is copying from image store to primary, in case it's an uploaded volume"),
Uploaded("Volume is uploaded"),
NotUploaded("The volume entry is just created in DB, not yet uploaded"),
UploadInProgress("Volume upload is in progress"),
UploadError("Volume upload encountered some error"),
UploadAbandoned("Volume upload is abandoned since the upload was never initiated within a specified time"),
Attaching("The volume is attaching to a VM from Ready state."),
Restoring("The volume is being restored from backup.");
Allocated(false, "The volume is allocated but has not been created yet."),
Creating(true, "The volume is being created. getPoolId() should reflect the pool where it is being created."),
Ready(false, "The volume is ready to be used."),
Migrating(true, "The volume is migrating to other storage pool"),
Snapshotting(true, "There is a snapshot created on this volume, not backed up to secondary storage yet"),
RevertSnapshotting(true, "There is a snapshot created on this volume, the volume is being reverting from snapshot"),
Resizing(true, "The volume is being resized"),
Expunging(true, "The volume is being expunging"),
Expunged(false, "The volume has been expunged, and can no longer be recovered"),
Destroy(false, "The volume is destroyed, and can be recovered."),
Destroying(false, "The volume is destroying, and can't be recovered."),
UploadOp(true, "The volume upload operation is in progress or in short the volume is on secondary storage"),
Copying(true, "Volume is copying from image store to primary, in case it's an uploaded volume"),
Uploaded(false, "Volume is uploaded"),
NotUploaded(true, "The volume entry is just created in DB, not yet uploaded"),
UploadInProgress(true, "Volume upload is in progress"),
UploadError(false, "Volume upload encountered some error"),
UploadAbandoned(false, "Volume upload is abandoned since the upload was never initiated within a specified time"),
Attaching(true, "The volume is attaching to a VM from Ready state."),
Restoring(true, "The volume is being restored from backup.");
boolean _transitional;
String _description;
private State(String description) {
/**
* Volume State
* @param transitional true for transition/non-final state, otherwise false
* @param description description of the state
*/
private State(boolean transitional, String description) {
_transitional = transitional;
_description = description;
}
@ -70,6 +78,10 @@ public interface Volume extends ControlledEntity, Identity, InternalIdentity, Ba
return s_fsm;
}
public boolean isTransitional() {
return _transitional;
}
public String getDescription() {
return _description;
}

3
engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/NetworkOrchestrationService.java
View File

@ -50,6 +50,7 @@ import com.cloud.network.rules.LoadBalancerContainer.Scheme;
import com.cloud.offering.NetworkOffering;
import com.cloud.user.Account;
import com.cloud.user.User;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.Pair;
import com.cloud.vm.Nic;
import com.cloud.vm.NicProfile;
@ -268,6 +269,8 @@ public interface NetworkOrchestrationService {
Map<String, String> finalizeServicesAndProvidersForNetwork(NetworkOffering offering, Long physicalNetworkId);
boolean stateTransitTo(Network network, Network.Event e) throws NoTransitionException;
List<Provider> getProvidersForServiceInNetwork(Network network, Service service);
StaticNatServiceProvider getStaticNatProviderForNetwork(Network network);

5
engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/VolumeService.java
View File

@ -18,13 +18,13 @@
*/
package org.apache.cloudstack.engine.subsystem.api.storage;
import com.cloud.agent.api.Answer;
import java.util.Map;
import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
import org.apache.cloudstack.framework.async.AsyncCallFuture;
import org.apache.cloudstack.storage.command.CommandResult;
import com.cloud.agent.api.Answer;
import com.cloud.agent.api.to.VirtualMachineTO;
import com.cloud.exception.StorageAccessException;
import com.cloud.host.Host;
@ -35,6 +35,9 @@ import com.cloud.user.Account;
import com.cloud.utils.Pair;
public interface VolumeService {
String SNAPSHOT_ID = "SNAPSHOT_ID";
class VolumeApiResult extends CommandResult {
private final VolumeInfo volume;

3
engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java
View File

@ -4424,7 +4424,8 @@ public class NetworkOrchestrator extends ManagerBase implements NetworkOrchestra
return accessDetails;
}
protected boolean stateTransitTo(final NetworkVO network, final Network.Event e) throws NoTransitionException {
@Override
public boolean stateTransitTo(final Network network, final Network.Event e) throws NoTransitionException {
return _stateMachine.transitTo(network, e, null, _networksDao);
}

2
engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeObject.java
View File

@ -851,7 +851,7 @@ public class VolumeObject implements VolumeInfo {
@Override
public boolean delete() {
return dataStore == null ? true : dataStore.delete(this);
return dataStore == null || dataStore.delete(this);
}
@Override

12
engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java
View File

@ -32,10 +32,6 @@ import java.util.concurrent.ExecutionException;
import javax.inject.Inject;
import org.apache.cloudstack.secret.dao.PassphraseDao;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.resource.StorageProcessor;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
import org.apache.cloudstack.engine.cloud.entity.api.VolumeEntity;
@ -66,6 +62,7 @@ import org.apache.cloudstack.framework.async.AsyncCallbackDispatcher;
import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
import org.apache.cloudstack.framework.async.AsyncRpcContext;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.cloudstack.secret.dao.PassphraseDao;
import org.apache.cloudstack.storage.RemoteHostEndPoint;
import org.apache.cloudstack.storage.command.CommandResult;
import org.apache.cloudstack.storage.command.CopyCmdAnswer;
@ -83,6 +80,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO;
import org.apache.cloudstack.storage.image.store.TemplateObject;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
@ -122,13 +120,16 @@ import com.cloud.storage.StoragePool;
import com.cloud.storage.VMTemplateStoragePoolVO;
import com.cloud.storage.VMTemplateStorageResourceAssoc;
import com.cloud.storage.VMTemplateStorageResourceAssoc.Status;
import com.cloud.storage.VMTemplateVO;
import com.cloud.storage.Volume;
import com.cloud.storage.Volume.State;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.VolumeVO;
import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.storage.dao.VMTemplatePoolDao;
import com.cloud.storage.dao.VolumeDao;
import com.cloud.storage.dao.VolumeDetailsDao;
import com.cloud.storage.resource.StorageProcessor;
import com.cloud.storage.snapshot.SnapshotApiService;
import com.cloud.storage.snapshot.SnapshotManager;
import com.cloud.storage.template.TemplateConstants;
@ -142,7 +143,6 @@ import com.cloud.utils.db.DB;
import com.cloud.utils.db.GlobalLock;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VirtualMachine;
import org.apache.commons.lang3.StringUtils;
@Component
public class VolumeServiceImpl implements VolumeService {
@ -206,8 +206,6 @@ public class VolumeServiceImpl implements VolumeService {
@Inject
private PassphraseDao passphraseDao;
private final static String SNAPSHOT_ID = "SNAPSHOT_ID";
public VolumeServiceImpl() {
}

143
framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java
View File

@ -38,9 +38,13 @@ import javax.naming.ConfigurationException;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.api.ApiErrorCode;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeService;
import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import org.apache.cloudstack.framework.jobs.AsyncJob;
@ -65,7 +69,12 @@ import org.apache.log4j.MDC;
import org.apache.log4j.NDC;
import com.cloud.cluster.ClusterManagerListener;
import com.cloud.network.Network;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.storage.Snapshot;
import com.cloud.storage.Volume;
import com.cloud.storage.VolumeDetailVO;
import com.cloud.storage.dao.SnapshotDao;
import com.cloud.storage.dao.SnapshotDetailsDao;
import com.cloud.storage.dao.SnapshotDetailsVO;
@ -93,7 +102,11 @@ import com.cloud.utils.db.TransactionCallbackNoReturn;
import com.cloud.utils.db.TransactionStatus;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.exception.ExceptionUtil;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.mgmt.JmxUtil;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineManager;
import com.cloud.vm.dao.VMInstanceDao;
public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, ClusterManagerListener, Configurable {
@ -148,6 +161,15 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
@Inject
private SnapshotDetailsDao _snapshotDetailsDao;
@Inject
private VolumeDataFactory volFactory;
@Inject
private VirtualMachineManager virtualMachineManager;
@Inject
private NetworkDao networkDao;
@Inject
private NetworkOrchestrationService networkOrchestrationService;
private volatile long _executionRunNumber = 1;
private final ScheduledExecutorService _heartbeatScheduler = Executors.newScheduledThreadPool(1, new NamedThreadFactory("AsyncJobMgr-Heartbeat"));
@ -1089,6 +1111,7 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
if (s_logger.isDebugEnabled()) {
s_logger.debug("Cancel left-over job-" + job.getId());
}
cleanupResources(job);
job.setStatus(JobInfo.Status.FAILED);
job.setResultCode(ApiErrorCode.INTERNAL_ERROR.getHttpCode());
job.setResult("job cancelled because of management server restart or shutdown");
@ -1101,26 +1124,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
s_logger.debug("Purge queue item for cancelled job-" + job.getId());
}
_queueMgr.purgeAsyncJobQueueItemId(job.getId());
if (ApiCommandResourceType.Volume.toString().equals(job.getInstanceType())) {
try {
_volumeDetailsDao.removeDetail(job.getInstanceId(), "SNAPSHOT_ID");
_volsDao.remove(job.getInstanceId());
} catch (Exception e) {
s_logger.error("Unexpected exception while removing concurrent request meta data :" + e.getLocalizedMessage());
}
}
}
final List<SnapshotDetailsVO> snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false);
for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) {
SnapshotInfo snapshot = snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId());
if (snapshot == null) {
_snapshotDetailsDao.remove(snapshotDetailsVO.getId());
continue;
}
snapshotSrv.processEventOnSnapshotObject(snapshot, Snapshot.Event.OperationFailed);
_snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(), AsyncJob.Constants.MS_ID);
}
cleanupFailedSnapshotsCreatedWithDefaultStrategy(msid);
}
});
} catch (Throwable e) {
@ -1128,6 +1133,106 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager,
}
}
/*
Cleanup Resources in transition state and move them to appropriate state
This will allow other operation on the resource, instead of being stuck in transition state
*/
protected boolean cleanupResources(AsyncJobVO job) {
try {
ApiCommandResourceType resourceType = ApiCommandResourceType.fromString(job.getInstanceType());
if (resourceType == null) {
s_logger.warn("Unknown ResourceType. Skip Cleanup: " + job.getInstanceType());
return true;
}
switch (resourceType) {
case Volume:
return cleanupVolume(job.getInstanceId());
case VirtualMachine:
return cleanupVirtualMachine(job.getInstanceId());
case Network:
return cleanupNetwork(job.getInstanceId());
}
} catch (Exception e) {
s_logger.warn("Error while cleaning up resource: [" + job.getInstanceType().toString() + "] with Id: " + job.getInstanceId(), e);
return false;
}
return true;
}
private boolean cleanupVolume(final long volumeId) {
VolumeInfo vol = volFactory.getVolume(volumeId);
if (vol == null) {
s_logger.warn("Volume not found. Skip Cleanup. VolumeId: " + volumeId);
return true;
}
if (vol.getState().isTransitional()) {
s_logger.debug("Cleaning up volume with Id: " + volumeId);
boolean status = vol.stateTransit(Volume.Event.OperationFailed);
cleanupFailedVolumesCreatedFromSnapshots(volumeId);
return status;
}
s_logger.debug("Volume not in transition state. Skip cleanup. VolumeId: " + volumeId);
return true;
}
private boolean cleanupVirtualMachine(final long vmId) throws Exception {
VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
if (vmInstanceVO == null) {
s_logger.warn("Instance not found. Skip Cleanup. InstanceId: " + vmId);
return true;
}
if (vmInstanceVO.getState().isTransitional()) {
s_logger.debug("Cleaning up Instance with Id: " + vmId);
return virtualMachineManager.stateTransitTo(vmInstanceVO, VirtualMachine.Event.OperationFailed, vmInstanceVO.getHostId());
}
s_logger.debug("Instance not in transition state. Skip cleanup. InstanceId: " + vmId);
return true;
}
private boolean cleanupNetwork(final long networkId) throws Exception {
NetworkVO networkVO = networkDao.findById(networkId);
if (networkVO == null) {
s_logger.warn("Network not found. Skip Cleanup. NetworkId: " + networkId);
return true;
}
if (Network.State.Implementing.equals(networkVO.getState())) {
try {
s_logger.debug("Cleaning up Network with Id: " + networkId);
return networkOrchestrationService.stateTransitTo(networkVO, Network.Event.OperationFailed);
} catch (final NoTransitionException e) {
networkVO.setState(Network.State.Shutdown);
networkDao.update(networkVO.getId(), networkVO);
}
}
s_logger.debug("Network not in transition state. Skip cleanup. NetworkId: " + networkId);
return true;
}
private void cleanupFailedVolumesCreatedFromSnapshots(final long volumeId) {
try {
VolumeDetailVO volumeDetail = _volumeDetailsDao.findDetail(volumeId, VolumeService.SNAPSHOT_ID);
if (volumeDetail != null) {
_volumeDetailsDao.removeDetail(volumeId, VolumeService.SNAPSHOT_ID);
_volsDao.remove(volumeId);
}
} catch (Exception e) {
s_logger.error("Unexpected exception while removing concurrent request meta data :" + e.getLocalizedMessage());
}
}
private void cleanupFailedSnapshotsCreatedWithDefaultStrategy(final long msid) {
final List<SnapshotDetailsVO> snapshotList = _snapshotDetailsDao.findDetails(AsyncJob.Constants.MS_ID, Long.toString(msid), false);
for (final SnapshotDetailsVO snapshotDetailsVO : snapshotList) {
SnapshotInfo snapshot = snapshotFactory.getSnapshotOnPrimaryStore(snapshotDetailsVO.getResourceId());
if (snapshot == null) {
_snapshotDetailsDao.remove(snapshotDetailsVO.getId());
continue;
}
snapshotSrv.processEventOnSnapshotObject(snapshot, Snapshot.Event.OperationFailed);
_snapshotDetailsDao.removeDetail(snapshotDetailsVO.getResourceId(), AsyncJob.Constants.MS_ID);
}
}
@Override
public void onManagementNodeJoined(List<? extends ManagementServerHost> nodeList, long selfNodeId) {
}

96
framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java Normal file
View File

@ -0,0 +1,96 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.cloudstack.framework.jobs.impl;
import com.cloud.network.Network;
import com.cloud.network.dao.NetworkDao;
import com.cloud.network.dao.NetworkVO;
import com.cloud.storage.Volume;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.vm.VMInstanceVO;
import com.cloud.vm.VirtualMachine;
import com.cloud.vm.VirtualMachineManager;
import com.cloud.vm.dao.VMInstanceDao;
import org.apache.cloudstack.api.ApiCommandResourceType;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory;
import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.MockitoJUnitRunner;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class AsyncJobManagerImplTest {
@Spy
@InjectMocks
AsyncJobManagerImpl asyncJobManager;
@Mock
VolumeDataFactory volFactory;
@Mock
VMInstanceDao vmInstanceDao;
@Mock
VirtualMachineManager virtualMachineManager;
@Mock
NetworkDao networkDao;
@Mock
NetworkOrchestrationService networkOrchestrationService;
@Test
public void testCleanupVolumeResource() {
AsyncJobVO job = new AsyncJobVO();
job.setInstanceType(ApiCommandResourceType.Volume.toString());
job.setInstanceId(1L);
VolumeInfo volumeInfo = Mockito.mock(VolumeInfo.class);
when(volFactory.getVolume(Mockito.anyLong())).thenReturn(volumeInfo);
when(volumeInfo.getState()).thenReturn(Volume.State.Attaching);
asyncJobManager.cleanupResources(job);
Mockito.verify(volumeInfo, Mockito.times(1)).stateTransit(Volume.Event.OperationFailed);
}
@Test
public void testCleanupVmResource() throws NoTransitionException {
AsyncJobVO job = new AsyncJobVO();
job.setInstanceType(ApiCommandResourceType.VirtualMachine.toString());
job.setInstanceId(1L);
VMInstanceVO vmInstanceVO = Mockito.mock(VMInstanceVO.class);
when(vmInstanceDao.findById(Mockito.anyLong())).thenReturn(vmInstanceVO);
when(vmInstanceVO.getState()).thenReturn(VirtualMachine.State.Starting);
when(vmInstanceVO.getHostId()).thenReturn(1L);
asyncJobManager.cleanupResources(job);
Mockito.verify(virtualMachineManager, Mockito.times(1)).stateTransitTo(vmInstanceVO, VirtualMachine.Event.OperationFailed, 1L);
}
@Test
public void testCleanupNetworkResource() throws NoTransitionException {
AsyncJobVO job = new AsyncJobVO();
job.setInstanceType(ApiCommandResourceType.Network.toString());
job.setInstanceId(1L);
NetworkVO networkVO = Mockito.mock(NetworkVO.class);
when(networkDao.findById(Mockito.anyLong())).thenReturn(networkVO);
when(networkVO.getState()).thenReturn(Network.State.Implementing);
asyncJobManager.cleanupResources(job);
Mockito.verify(networkOrchestrationService, Mockito.times(1)).stateTransitTo(networkVO,
Network.Event.OperationFailed);
}
}

6
server/src/test/java/com/cloud/vpc/MockNetworkManagerImpl.java
View File

@ -26,6 +26,7 @@ import javax.naming.ConfigurationException;
import com.cloud.dc.DataCenter;
import com.cloud.network.PublicIpQuarantine;
import com.cloud.utils.fsm.NoTransitionException;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.api.command.admin.address.ReleasePodIpCmdByAdmin;
import org.apache.cloudstack.api.command.admin.network.DedicateGuestVlanRangeCmd;
@ -824,6 +825,11 @@ public class MockNetworkManagerImpl extends ManagerBase implements NetworkOrches
return null;
}
@Override
public boolean stateTransitTo(Network network, Network.Event e) throws NoTransitionException {
return true;
}
@Override
public boolean isNetworkInlineMode(Network network) {
// TODO Auto-generated method stub