diff --git a/api/src/main/java/com/cloud/agent/api/Command.java b/api/src/main/java/com/cloud/agent/api/Command.java index b3c6120462a..c873139099c 100644 --- a/api/src/main/java/com/cloud/agent/api/Command.java +++ b/api/src/main/java/com/cloud/agent/api/Command.java @@ -47,6 +47,13 @@ public abstract class Command { return wait; } + /** + * This is the time in seconds that the agent will wait before waiting for an answer from the endpoint. + * The actual wait time is twice the value of this variable. + * See {@link com.cloud.agent.manager.AgentAttache#send(com.cloud.agent.transport.Request, int) AgentAttache#send} implementation for more details. + * + * @param wait + **/ public void setWait(int wait) { this.wait = wait; } diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index b3d9b44ec7f..1ac8d79b9d5 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -395,6 +395,8 @@ public class EventTypes { public static final String EVENT_STORAGE_IP_RANGE_UPDATE = "STORAGE.IP.RANGE.UPDATE"; public static final String EVENT_IMAGE_STORE_DATA_MIGRATE = "IMAGE.STORE.MIGRATE.DATA"; + public static final String EVENT_IMAGE_STORE_RESOURCES_MIGRATE = "IMAGE.STORE.MIGRATE.RESOURCES"; + public static final String EVENT_IMAGE_STORE_OBJECT_DOWNLOAD = "IMAGE.STORE.OBJECT.DOWNLOAD"; // Configuration Table public static final String EVENT_CONFIGURATION_VALUE_EDIT = "CONFIGURATION.VALUE.EDIT"; @@ -1147,6 +1149,7 @@ public class EventTypes { entityEventDetails.put(EVENT_IMPORT_VCENTER_STORAGE_POLICIES, "StoragePolicies"); entityEventDetails.put(EVENT_IMAGE_STORE_DATA_MIGRATE, ImageStore.class); + entityEventDetails.put(EVENT_IMAGE_STORE_OBJECT_DOWNLOAD, ImageStore.class); entityEventDetails.put(EVENT_LIVE_PATCH_SYSTEMVM, "SystemVMs"); } diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index b65485dcfe4..a6f27d1469a 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -972,6 +972,7 @@ public class ApiConstants { public static final String TARGET_ID = "targetid"; public static final String FILES = "files"; public static final String SRC_POOL = "srcpool"; + public static final String DEST_POOL = "destpool"; public static final String DEST_POOLS = "destpools"; public static final String VOLUME_IDS = "volumeids"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdmin.java index 4b6d4c0bd58..0719d2a7ce4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdmin.java @@ -17,12 +17,33 @@ package org.apache.cloudstack.api.command.admin.iso; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.TemplateResponse; @APICommand(name = "listIsos", description = "Lists all available ISO files.", responseObject = TemplateResponse.class, responseView = ResponseView.Full, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListIsosCmdByAdmin extends ListIsosCmd implements AdminCmd { + @Parameter(name = ApiConstants.IMAGE_STORE_ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, + description = "ID of the image or image cache store", since = "4.19") + private Long imageStoreId; + + @Parameter(name = ApiConstants.STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, + description = "ID of the storage pool", since = "4.19") + private Long storagePoolId; + + @Override + public Long getImageStoreId() { + return imageStoreId; + } + + @Override + public Long getStoragePoolId() { + return storagePoolId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/snapshot/ListSnapshotsCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/snapshot/ListSnapshotsCmdByAdmin.java new file mode 100644 index 00000000000..7070ba349e6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/snapshot/ListSnapshotsCmdByAdmin.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.snapshot; + +import com.cloud.storage.Snapshot; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; +import org.apache.cloudstack.api.command.admin.AdminCmd; +import org.apache.cloudstack.api.command.user.snapshot.ListSnapshotsCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; + +@APICommand(name = "listSnapshots", description = "Lists all available snapshots for the account.", responseObject = SnapshotResponse.class, entityType = { + Snapshot.class }, responseView = ResponseObject.ResponseView.Full, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +public class ListSnapshotsCmdByAdmin extends ListSnapshotsCmd implements AdminCmd { + @Parameter(name = ApiConstants.IMAGE_STORE_ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, + description = "ID of the image or image cache store", since = "4.19") + private Long imageStoreId; + + @Parameter(name = ApiConstants.STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, + description = "ID of the storage pool", since = "4.19") + private Long storagePoolId; + + @Override + public Long getImageStoreId() { + return imageStoreId; + } + + @Override + public Long getStoragePoolId() { + return storagePoolId; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmd.java new file mode 100644 index 00000000000..92019e70eca --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmd.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.event.EventTypes; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ExtractResponse; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.storage.browser.StorageBrowser; + +import javax.inject.Inject; +import java.nio.file.Path; + +@APICommand(name = "downloadImageStoreObject", description = "Download object at a specified path on an image store.", + responseObject = ExtractResponse.class, since = "4.19.0", requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false) +public class DownloadImageStoreObjectCmd extends BaseAsyncCmd { + + @Inject + StorageBrowser storageBrowser; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, required = true, + description = "id of the image store") + private Long storeId; + + @Parameter(name = ApiConstants.PATH, type = CommandType.STRING, description = "path to download on image store") + private String path; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getStoreId() { + return storeId; + } + + public String getPath() { + if (path == null) { + path = "/"; + } + // We prepend "/" to path and normalize to prevent path traversal attacks + return Path.of(String.format("/%s", path)).normalize().toString().substring(1); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ExtractResponse response = storageBrowser.downloadImageStoreObject(this); + response.setResponseName(getCommandName()); + response.setObjectName(getCommandName()); + this.setResponseObject(response); + } + + /** + * For commands the API framework needs to know the owner of the object being acted upon. This method is + * used to determine that information. + * + * @return the id of the account that owns the object being acted upon + */ + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccount().getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_IMAGE_STORE_OBJECT_DOWNLOAD; + } + + @Override + public String getEventDescription() { + return "Downloading object at path " + getPath() + " on image store " + getStoreId(); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoreObjectsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoreObjectsCmd.java new file mode 100644 index 00000000000..48fd25c99c6 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoreObjectsCmd.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; +import org.apache.cloudstack.storage.browser.StorageBrowser; + +import javax.inject.Inject; +import java.nio.file.Path; + +@APICommand(name = "listImageStoreObjects", description = "Lists objects at specified path on an image store.", + responseObject = DataStoreObjectResponse.class, since = "4.19.0", requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false) +public class ListImageStoreObjectsCmd extends BaseListCmd { + + @Inject + StorageBrowser storageBrowser; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, required = true, + description = "id of the image store") + private Long storeId; + + @Parameter(name = ApiConstants.PATH, type = CommandType.STRING, description = "path to list on image store") + private String path; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getStoreId() { + return storeId; + } + + public String getPath() { + if (path == null) { + path = "/"; + } + // We prepend "/" to path and normalize to prevent path traversal attacks + return Path.of(String.format("/%s", path)).normalize().toString().substring(1); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = storageBrowser.listImageStoreObjects(this); + response.setResponseName(getCommandName()); + response.setObjectName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolObjectsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolObjectsCmd.java new file mode 100644 index 00000000000..4dac92a9572 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolObjectsCmd.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseListCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; +import org.apache.cloudstack.storage.browser.StorageBrowser; + +import javax.inject.Inject; +import java.nio.file.Path; + + +@APICommand(name = "listStoragePoolObjects", description = "Lists objects at specified path on a storage pool.", + responseObject = DataStoreObjectResponse.class, since = "4.19.0", requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false) +public class ListStoragePoolObjectsCmd extends BaseListCmd { + + @Inject + StorageBrowser storageBrowser; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, required = true, + description = "id of the storage pool") + private Long storeId; + + @Parameter(name = ApiConstants.PATH, type = CommandType.STRING, description = "path to list on storage pool") + private String path; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getStoreId() { + return storeId; + } + + public String getPath() { + if (path == null) { + path = "/"; + } + // We prepend "/" to path and normalize to prevent path traversal attacks + return Path.of(String.format("/%s", path)).normalize().toString().substring(1); + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() { + ListResponse response = storageBrowser.listPrimaryStoreObjects(this); + response.setResponseName(getCommandName()); + response.setObjectName(getCommandName()); + this.setResponseObject(response); + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateResourcesToAnotherSecondaryStorageCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateResourcesToAnotherSecondaryStorageCmd.java new file mode 100644 index 00000000000..3bc4fd7a371 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/storage/MigrateResourcesToAnotherSecondaryStorageCmd.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.event.EventTypes; +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.MigrationResponse; +import org.apache.cloudstack.api.response.SnapshotResponse; +import org.apache.cloudstack.api.response.TemplateResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.commons.collections.CollectionUtils; + +import java.util.Collections; +import java.util.List; + +@APICommand(name = "migrateResourceToAnotherSecondaryStorage", + description = "migrates resources from one secondary storage to destination image store", + responseObject = MigrationResponse.class, + requestHasSensitiveInfo = false, + responseHasSensitiveInfo = false, + since = "4.19.0", + authorized = {RoleType.Admin}) +public class MigrateResourcesToAnotherSecondaryStorageCmd extends BaseAsyncCmd { + + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.SRC_POOL, + type = CommandType.UUID, + entityType = ImageStoreResponse.class, + description = "id of the image store from where the data is to be migrated", + required = true) + private Long id; + + + @Parameter(name = ApiConstants.DEST_POOL, + type = CommandType.UUID, + entityType = ImageStoreResponse.class, + description = "id of the destination secondary storage pool to which the resources are to be migrated", + required = true) + private Long destStoreId; + + @Parameter(name = "templates", + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType = TemplateResponse.class, + description = "id(s) of the templates to be migrated", + required = false) + private List templateIdList; + + @Parameter(name = "snapshots", + type = CommandType.LIST, + collectionType = CommandType.UUID, + entityType = SnapshotResponse.class, + description = "id(s) of the snapshots to be migrated", + required = false) + private List snapshotIdList; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getDestStoreId() { + return destStoreId; + } + + public List getTemplateIdList() { + if (CollectionUtils.isEmpty(templateIdList)) { + return Collections.emptyList(); + } + return templateIdList; + } + + public List getSnapshotIdList() { + if (CollectionUtils.isEmpty(snapshotIdList)) { + return Collections.emptyList(); + } + return snapshotIdList; + } + + @Override + public String getEventType() { + return EventTypes.EVENT_IMAGE_STORE_RESOURCES_MIGRATE; + } + + @Override + public String getEventDescription() { + return "Attempting to migrate files/data objects to another image store"; + } + + @Override + public void execute() { + MigrationResponse response = _imageStoreService.migrateResources(this); + response.setObjectName("imagestore"); + this.setResponseObject(response); + CallContext.current().setEventDetails(response.getMessage()); + } + + @Override + public long getEntityOwnerId() { + return CallContext.current().getCallingAccountId(); + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + public Long getId() { + return id; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.ImageStore; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdmin.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdmin.java index 2f57783e5ea..22eb0ff0659 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdmin.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdmin.java @@ -17,15 +17,36 @@ package org.apache.cloudstack.api.command.admin.template; import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.Parameter; import org.apache.cloudstack.api.ResponseObject.ResponseView; import org.apache.cloudstack.api.command.admin.AdminCmd; import org.apache.cloudstack.api.command.user.template.ListTemplatesCmd; +import org.apache.cloudstack.api.response.ImageStoreResponse; +import org.apache.cloudstack.api.response.StoragePoolResponse; import org.apache.cloudstack.api.response.TemplateResponse; import com.cloud.template.VirtualMachineTemplate; -@APICommand(name = "listTemplates", description = "List all public, private, and privileged templates.", responseObject = TemplateResponse.class, entityType = {VirtualMachineTemplate.class}, responseView = ResponseView.Full, - requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) +@APICommand(name = "listTemplates", description = "List all public, private, and privileged templates.", + responseObject = TemplateResponse.class, entityType = {VirtualMachineTemplate.class}, + responseView = ResponseView.Full, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListTemplatesCmdByAdmin extends ListTemplatesCmd implements AdminCmd { + @Parameter(name = ApiConstants.IMAGE_STORE_ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, + description = "ID of the image or image cache store", since = "4.19") + private Long imageStoreId; + @Parameter(name = ApiConstants.STORAGE_ID, type = CommandType.UUID, entityType = StoragePoolResponse.class, + description = "ID of the storage pool", since = "4.19") + private Long storagePoolId; + + @Override + public Long getImageStoreId() { + return imageStoreId; + } + + @Override + public Long getStoragePoolId() { + return storagePoolId; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java index 90ecaa47de0..f723cb93ae9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmd.java @@ -134,6 +134,10 @@ public class ListIsosCmd extends BaseListTaggedResourcesCmd implements UserCmd { return showUnique != null && showUnique; } + public Long getImageStoreId() { + return null; + } + public Boolean getShowIcon () { return showIcon != null ? showIcon : false; } @@ -191,4 +195,8 @@ public class ListIsosCmd extends BaseListTaggedResourcesCmd implements UserCmd { templateResponse.setResourceIconResponse(iconResponse); } } + + public Long getStoragePoolId() { + return null; + }; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java index 23515284e4c..cf665127a17 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/snapshot/ListSnapshotsCmd.java @@ -23,6 +23,7 @@ import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseListTaggedResourcesCmd; import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ResponseObject; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.api.response.SnapshotResponse; import org.apache.cloudstack.api.response.VolumeResponse; @@ -32,7 +33,7 @@ import org.apache.log4j.Logger; import com.cloud.storage.Snapshot; @APICommand(name = "listSnapshots", description = "Lists all available snapshots for the account.", responseObject = SnapshotResponse.class, entityType = { - Snapshot.class }, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) + Snapshot.class }, responseView = ResponseObject.ResponseView.Restricted, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class ListSnapshotsCmd extends BaseListTaggedResourcesCmd { public static final Logger s_logger = Logger.getLogger(ListSnapshotsCmd.class.getName()); @@ -111,6 +112,14 @@ public class ListSnapshotsCmd extends BaseListTaggedResourcesCmd { return null; } + public Long getImageStoreId() { + return null; + } + + public Long getStoragePoolId() { + return null; + } + ///////////////////////////////////////////////////// /////////////// API Implementation/////////////////// ///////////////////////////////////////////////////// diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java index 26d79084531..dae7cc97a4c 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmd.java @@ -24,6 +24,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import org.apache.cloudstack.api.APICommand; @@ -224,6 +225,17 @@ public class ListTemplatesCmd extends BaseListTaggedResourcesCmd implements User } public List getIds() { + if (ids == null) { + return Collections.emptyList(); + } return ids; } + + public Long getImageStoreId() { + return null; + } + + public Long getStoragePoolId() { + return null; + } } diff --git a/api/src/main/java/org/apache/cloudstack/storage/ImageStoreObjectDownload.java b/api/src/main/java/org/apache/cloudstack/storage/ImageStoreObjectDownload.java new file mode 100644 index 00000000000..02bce11dfa4 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/ImageStoreObjectDownload.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage; + +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +public interface ImageStoreObjectDownload extends InternalIdentity { + + long getId(); + + Long getStoreId(); + + String getPath(); + + String getDownloadUrl(); + + Date getCreated(); +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java b/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java index b8f14ad2bfa..9dd54dc4336 100644 --- a/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java +++ b/api/src/main/java/org/apache/cloudstack/storage/ImageStoreService.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.storage; +import org.apache.cloudstack.api.command.admin.storage.MigrateResourcesToAnotherSecondaryStorageCmd; import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; import org.apache.cloudstack.api.response.MigrationResponse; @@ -26,4 +27,5 @@ public interface ImageStoreService { BALANCE, COMPLETE } MigrationResponse migrateData(MigrateSecondaryStorageDataCmd cmd); + MigrationResponse migrateResources(MigrateResourcesToAnotherSecondaryStorageCmd migrateResourcesToAnotherSecondaryStorageCmd); } diff --git a/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java b/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java new file mode 100644 index 00000000000..cac5cc91b03 --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/browser/DataStoreObjectResponse.java @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.storage.browser; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.BaseResponse; + +import java.util.Date; + +public class DataStoreObjectResponse extends BaseResponse { + + @SerializedName(ApiConstants.NAME) + @Param(description = "Name of the data store object.") + private String name; + + @SerializedName("isdirectory") + @Param(description = "Is it a directory.") + private boolean isDirectory; + + @SerializedName(ApiConstants.SIZE) + @Param(description = "Size is in Bytes.") + private long size; + + @SerializedName(ApiConstants.TEMPLATE_ID) + @Param(description = "Template ID associated with the data store object.") + private String templateId; + + @SerializedName(ApiConstants.FORMAT) + @Param(description = "Format of template associated with the data store object.") + private String format; + + @SerializedName(ApiConstants.SNAPSHOT_ID) + @Param(description = "Snapshot ID associated with the data store object.") + private String snapshotId; + + @SerializedName(ApiConstants.VOLUME_ID) + @Param(description = "Volume ID associated with the data store object.") + private String volumeId; + + @SerializedName(ApiConstants.LAST_UPDATED) + @Param(description = "Last modified date of the file/directory.") + private Date lastUpdated; + + public DataStoreObjectResponse(String name, boolean isDirectory, long size, Date lastUpdated) { + this.name = name; + this.isDirectory = isDirectory; + this.size = size; + this.lastUpdated = lastUpdated; + this.setObjectName("datastoreobject"); + } + + public DataStoreObjectResponse() { + super(); + this.setObjectName("datastoreobject"); + } + + public void setTemplateId(String templateId) { + this.templateId = templateId; + } + + public void setFormat(String format) { + this.format = format; + } + + public void setSnapshotId(String snapshotId) { + this.snapshotId = snapshotId; + } + + public void setVolumeId(String volumeId) { + this.volumeId = volumeId; + } + + public String getName() { + return name; + } + + public boolean isDirectory() { + return isDirectory; + } + + public long getSize() { + return size; + } + + public String getTemplateId() { + return templateId; + } + + public String getFormat() { + return format; + } + + public String getSnapshotId() { + return snapshotId; + } + + public String getVolumeId() { + return volumeId; + } + + public Date getLastUpdated() { + return lastUpdated; + } +} diff --git a/api/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowser.java b/api/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowser.java new file mode 100644 index 00000000000..b9fe5b07fec --- /dev/null +++ b/api/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowser.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.browser; + +import com.cloud.utils.component.PluggableService; +import org.apache.cloudstack.api.command.admin.storage.DownloadImageStoreObjectCmd; +import org.apache.cloudstack.api.command.admin.storage.ListImageStoreObjectsCmd; +import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolObjectsCmd; +import org.apache.cloudstack.api.response.ExtractResponse; +import org.apache.cloudstack.api.response.ListResponse; + +public interface StorageBrowser extends PluggableService { + ListResponse listImageStoreObjects(ListImageStoreObjectsCmd cmd); + + ListResponse listPrimaryStoreObjects(ListStoragePoolObjectsCmd cmd); + + ExtractResponse downloadImageStoreObject(DownloadImageStoreObjectCmd cmd); +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdminTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdminTest.java new file mode 100644 index 00000000000..943db0cdfb7 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/iso/ListIsosCmdByAdminTest.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.iso; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.Assert.assertEquals; + +public class ListIsosCmdByAdminTest { + + @InjectMocks + ListIsosCmdByAdmin cmd; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetImageStoreId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "imageStoreId", id); + assertEquals(id, cmd.getImageStoreId()); + } + + @Test + public void testGetZoneId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "zoneId", id); + assertEquals(id, cmd.getZoneId()); + } + + @Test + public void testGetShowRemoved() { + Boolean showRemoved = true; + ReflectionTestUtils.setField(cmd, "showRemoved", showRemoved); + assertEquals(showRemoved, cmd.getShowRemoved()); + } + + @Test + public void testGetIsoName() { + String isoName = "test"; + ReflectionTestUtils.setField(cmd, "isoName", isoName); + assertEquals(isoName, cmd.getIsoName()); + } + + @Test + public void testGetIsoFilter() { + String isoFilter = "test"; + ReflectionTestUtils.setField(cmd, "isoFilter", isoFilter); + assertEquals(isoFilter, cmd.getIsoFilter()); + } + + @Test + public void testGetShowUnique() { + Boolean showUnique = true; + ReflectionTestUtils.setField(cmd, "showUnique", showUnique); + assertEquals(showUnique, cmd.getShowUnique()); + } + + @Test + public void testGetShowIcon() { + Boolean showIcon = true; + ReflectionTestUtils.setField(cmd, "showIcon", showIcon); + assertEquals(showIcon, cmd.getShowIcon()); + } + + @Test + public void testGetBootable() { + Boolean bootable = true; + ReflectionTestUtils.setField(cmd, "bootable", bootable); + assertEquals(bootable, cmd.isBootable()); + } + + @Test + public void testGetHypervisor() { + String hypervisor = "test"; + ReflectionTestUtils.setField(cmd, "hypervisor", hypervisor); + assertEquals(hypervisor, cmd.getHypervisor()); + } + + @Test + public void testGetId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "id", id); + assertEquals(id, cmd.getId()); + } + + @Test + public void testGetPublic() { + Boolean publicIso = true; + ReflectionTestUtils.setField(cmd, "publicIso", publicIso); + assertEquals(publicIso, cmd.isPublic()); + } + + @Test + public void testGetReady() { + Boolean ready = true; + ReflectionTestUtils.setField(cmd, "ready", ready); + assertEquals(ready, cmd.isReady()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java new file mode 100644 index 00000000000..98435bf6e38 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/DownloadImageStoreObjectCmdTest.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.utils.Pair; +import org.apache.cloudstack.api.response.ExtractResponse; +import org.apache.cloudstack.storage.browser.StorageBrowser; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.List; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class DownloadImageStoreObjectCmdTest { + + @Mock + private StorageBrowser storageBrowser; + + @InjectMocks + @Spy + private DownloadImageStoreObjectCmd cmd; + + private AutoCloseable closeable; + + @Before + public void setUp() { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testExecute() throws Exception { + ReflectionTestUtils.setField(cmd, "storeId", 1L); + ReflectionTestUtils.setField(cmd, "path", "path/to/object"); + ExtractResponse response = mock(ExtractResponse.class); + when(storageBrowser.downloadImageStoreObject(cmd)).thenReturn(response); + + cmd.execute(); + + verify(storageBrowser).downloadImageStoreObject(cmd); + verify(response).setResponseName("downloadImageStoreObjectResponse".toLowerCase()); + verify(response).setObjectName("downloadImageStoreObjectResponse".toLowerCase()); + verify(cmd).setResponseObject(response); + } + + @Test + public void testGetPath() { + List> pair = List.of( + new Pair<>("", null), + new Pair<>("", ""), + new Pair<>("", "/"), + new Pair<>("etc", "etc"), + new Pair<>("etc", "/etc"), + new Pair<>("etc/passwd", "etc/passwd"), + new Pair<>("etc/passwd", "//etc/passwd"), + new Pair<>("", "/etc/passwd/../../.."), + new Pair<>("etc/passwd", "../../etc/passwd"), + new Pair<>("etc/passwd", "/../../etc/passwd"), + new Pair<>("etc/passwd", ";../../../etc/passwd"), + new Pair<>("etc/passwd", "///etc/passwd"), + new Pair<>("etc/passwd", "/abc/xyz/../../../etc/passwd") + ); + + for (Pair p : pair) { + String expectedPath = p.first(); + String path = p.second(); + ReflectionTestUtils.setField(cmd, "path", path); + Assert.assertEquals(expectedPath, cmd.getPath()); + } + } + + @Test + public void testGetEventType() { + String eventType = cmd.getEventType(); + + Assert.assertEquals("IMAGE.STORE.OBJECT.DOWNLOAD", eventType); + } + + @Test + public void testGetEventDescription() { + ReflectionTestUtils.setField(cmd, "storeId", 1L); + ReflectionTestUtils.setField(cmd, "path", "path/to/object"); + String eventDescription = cmd.getEventDescription(); + + Assert.assertEquals("Downloading object at path path/to/object on image store 1", eventDescription); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoreObjectsCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoreObjectsCmdTest.java new file mode 100644 index 00000000000..88dcde1a4b0 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/ListImageStoreObjectsCmdTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.utils.Pair; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; +import org.apache.cloudstack.storage.browser.StorageBrowser; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.List; + +public class ListImageStoreObjectsCmdTest { + + @Mock + StorageBrowser storageBrowser; + + @InjectMocks + private ListImageStoreObjectsCmd cmd = new ListImageStoreObjectsCmd(); + + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetStoreId() { + Long id = 123L; + ReflectionTestUtils.setField(cmd, "storeId", id); + Assert.assertEquals(id, cmd.getStoreId()); + } + + @Test + public void testGetPath() { + List> pair = List.of( + new Pair<>("", null), + new Pair<>("", ""), + new Pair<>("", "/"), + new Pair<>("etc", "etc"), + new Pair<>("etc", "/etc"), + new Pair<>("etc/passwd", "etc/passwd"), + new Pair<>("etc/passwd", "//etc/passwd"), + new Pair<>("", "/etc/passwd/../../.."), + new Pair<>("etc/passwd", "../../etc/passwd"), + new Pair<>("etc/passwd", "/../../etc/passwd"), + new Pair<>("etc/passwd", ";../../../etc/passwd"), + new Pair<>("etc/passwd", "///etc/passwd"), + new Pair<>("etc/passwd", "/abc/xyz/../../../etc/passwd") + ); + + for (Pair p : pair) { + String expectedPath = p.first(); + String path = p.second(); + ReflectionTestUtils.setField(cmd, "path", path); + Assert.assertEquals(expectedPath, cmd.getPath()); + } + } + + @Test + public void testSuccessfulExecution() { + ListResponse response = Mockito.mock(ListResponse.class); + Mockito.when(storageBrowser.listImageStoreObjects(cmd)).thenReturn(response); + cmd.execute(); + Assert.assertEquals(response, cmd.getResponseObject()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolObjectsCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolObjectsCmdTest.java new file mode 100644 index 00000000000..a15740da819 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/ListStoragePoolObjectsCmdTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.api.command.admin.storage; + +import com.cloud.utils.Pair; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.storage.browser.DataStoreObjectResponse; +import org.apache.cloudstack.storage.browser.StorageBrowser; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.List; + +public class ListStoragePoolObjectsCmdTest { + @Mock + StorageBrowser storageBrowser; + + @InjectMocks + private ListStoragePoolObjectsCmd cmd = new ListStoragePoolObjectsCmd(); + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetStoreId() { + Long id = 123L; + ReflectionTestUtils.setField(cmd, "storeId", id); + Assert.assertEquals(id, cmd.getStoreId()); + } + + @Test + public void testGetPath() { + List> pair = List.of( + new Pair<>("", null), + new Pair<>("", ""), + new Pair<>("", "/"), + new Pair<>("etc", "etc"), + new Pair<>("etc", "/etc"), + new Pair<>("etc/passwd", "etc/passwd"), + new Pair<>("etc/passwd", "//etc/passwd"), + new Pair<>("", "/etc/passwd/../../.."), + new Pair<>("etc/passwd", "../../etc/passwd"), + new Pair<>("etc/passwd", "/../../etc/passwd"), + new Pair<>("etc/passwd", ";../../../etc/passwd"), + new Pair<>("etc/passwd", "///etc/passwd"), + new Pair<>("etc/passwd", "/abc/xyz/../../../etc/passwd") + ); + + for (Pair p : pair) { + String expectedPath = p.first(); + String path = p.second(); + ReflectionTestUtils.setField(cmd, "path", path); + Assert.assertEquals(expectedPath, cmd.getPath()); + } + } + + @Test + public void testSuccessfulExecution() { + ListResponse response = Mockito.mock(ListResponse.class); + Mockito.when(storageBrowser.listPrimaryStoreObjects(cmd)).thenReturn(response); + cmd.execute(); + Assert.assertEquals(response, cmd.getResponseObject()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/MigrateResourcesToAnotherSecondaryStorageCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/MigrateResourcesToAnotherSecondaryStorageCmdTest.java new file mode 100644 index 00000000000..2f000ee2b42 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/storage/MigrateResourcesToAnotherSecondaryStorageCmdTest.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.storage; + +import org.apache.cloudstack.api.response.MigrationResponse; +import org.apache.cloudstack.storage.ImageStoreService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import java.util.List; + +public class MigrateResourcesToAnotherSecondaryStorageCmdTest { + + + @Mock + ImageStoreService _imageStoreService; + + @InjectMocks + MigrateResourcesToAnotherSecondaryStorageCmd cmd; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetDestStoreId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "destStoreId", id); + Assert.assertEquals(id, cmd.getDestStoreId()); + } + + @Test + public void testGetId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "id", id); + Assert.assertEquals(id, cmd.getId()); + } + + @Test + public void testGetTemplateIdList() { + List ids = List.of(1234L, 5678L); + ReflectionTestUtils.setField(cmd, "templateIdList", ids); + Assert.assertEquals(ids, cmd.getTemplateIdList()); + } + + @Test + public void testGetSnapshotIdList() { + List ids = List.of(1234L, 5678L); + ReflectionTestUtils.setField(cmd, "snapshotIdList", ids); + Assert.assertEquals(ids, cmd.getSnapshotIdList()); + } + + @Test + public void testExecute() { + MigrationResponse response = Mockito.mock(MigrationResponse.class); + Mockito.when(_imageStoreService.migrateResources(Mockito.any())).thenReturn(response); + cmd.execute(); + Assert.assertEquals(response, cmd.getResponseObject()); + } + +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdminTest.java b/api/src/test/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdminTest.java new file mode 100644 index 00000000000..bdb14964eee --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/admin/template/ListTemplatesCmdByAdminTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.admin.template; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.Assert.assertEquals; + +public class ListTemplatesCmdByAdminTest { + + @InjectMocks + ListTemplatesCmdByAdmin cmd; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetImageStoreId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "imageStoreId", id); + assertEquals(id, cmd.getImageStoreId()); + } + + @Test + public void testGetZoneId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "zoneId", id); + assertEquals(id, cmd.getZoneId()); + } + + @Test + public void testGetShowRemoved() { + Boolean showRemoved = true; + ReflectionTestUtils.setField(cmd, "showRemoved", showRemoved); + assertEquals(showRemoved, cmd.getShowRemoved()); + } + + @Test + public void testGetShowUnique() { + Boolean showUnique = true; + ReflectionTestUtils.setField(cmd, "showUnique", showUnique); + assertEquals(showUnique, cmd.getShowUnique()); + } + + @Test + public void testGetShowIcon() { + Boolean showIcon = true; + ReflectionTestUtils.setField(cmd, "showIcon", showIcon); + assertEquals(showIcon, cmd.getShowIcon()); + } + + @Test + public void testGetHypervisor() { + String hypervisor = "test"; + ReflectionTestUtils.setField(cmd, "hypervisor", hypervisor); + assertEquals(hypervisor, cmd.getHypervisor()); + } + + @Test + public void testGetId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "id", id); + assertEquals(id, cmd.getId()); + } + +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmdTest.java new file mode 100644 index 00000000000..cc8a31f3ea8 --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/iso/ListIsosCmdTest.java @@ -0,0 +1,128 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.iso; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class ListIsosCmdTest { + + @InjectMocks + ListIsosCmd cmd; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetImageStoreId() { + ListIsosCmd cmd = new ListIsosCmd(); + assertNull(cmd.getImageStoreId()); + } + + @Test + public void testGetZoneId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "zoneId", id); + assertEquals(id, cmd.getZoneId()); + } + + @Test + public void testGetShowRemoved() { + Boolean showRemoved = true; + ReflectionTestUtils.setField(cmd, "showRemoved", showRemoved); + assertEquals(showRemoved, cmd.getShowRemoved()); + } + + @Test + public void testGetIsoName() { + String isoName = "test"; + ReflectionTestUtils.setField(cmd, "isoName", isoName); + assertEquals(isoName, cmd.getIsoName()); + } + + @Test + public void testGetIsoFilter() { + String isoFilter = "test"; + ReflectionTestUtils.setField(cmd, "isoFilter", isoFilter); + assertEquals(isoFilter, cmd.getIsoFilter()); + } + + @Test + public void testGetShowUnique() { + Boolean showUnique = true; + ReflectionTestUtils.setField(cmd, "showUnique", showUnique); + assertEquals(showUnique, cmd.getShowUnique()); + } + + @Test + public void testGetShowIcon() { + Boolean showIcon = true; + ReflectionTestUtils.setField(cmd, "showIcon", showIcon); + assertEquals(showIcon, cmd.getShowIcon()); + } + + @Test + public void testGetBootable() { + Boolean bootable = true; + ReflectionTestUtils.setField(cmd, "bootable", bootable); + assertEquals(bootable, cmd.isBootable()); + } + + @Test + public void testGetHypervisor() { + String hypervisor = "test"; + ReflectionTestUtils.setField(cmd, "hypervisor", hypervisor); + assertEquals(hypervisor, cmd.getHypervisor()); + } + + @Test + public void testGetId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "id", id); + assertEquals(id, cmd.getId()); + } + + @Test + public void testGetPublic() { + Boolean publicIso = true; + ReflectionTestUtils.setField(cmd, "publicIso", publicIso); + assertEquals(publicIso, cmd.isPublic()); + } + + @Test + public void testGetReady() { + Boolean ready = true; + ReflectionTestUtils.setField(cmd, "ready", ready); + assertEquals(ready, cmd.isReady()); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmdTest.java b/api/src/test/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmdTest.java new file mode 100644 index 00000000000..c48ca47db7c --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/command/user/template/ListTemplatesCmdTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.api.command.user.template; + +import org.apache.cloudstack.api.command.user.iso.ListIsosCmd; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.MockitoAnnotations; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class ListTemplatesCmdTest { + + @InjectMocks + ListTemplatesCmd cmd; + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetImageStoreId() { + ListIsosCmd cmd = new ListIsosCmd(); + assertNull(cmd.getImageStoreId()); + } + + @Test + public void testGetZoneId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "zoneId", id); + assertEquals(id, cmd.getZoneId()); + } + + @Test + public void testGetShowRemoved() { + Boolean showRemoved = true; + ReflectionTestUtils.setField(cmd, "showRemoved", showRemoved); + assertEquals(showRemoved, cmd.getShowRemoved()); + } + + @Test + public void testGetShowUnique() { + Boolean showUnique = true; + ReflectionTestUtils.setField(cmd, "showUnique", showUnique); + assertEquals(showUnique, cmd.getShowUnique()); + } + + @Test + public void testGetShowIcon() { + Boolean showIcon = true; + ReflectionTestUtils.setField(cmd, "showIcon", showIcon); + assertEquals(showIcon, cmd.getShowIcon()); + } + + @Test + public void testGetHypervisor() { + String hypervisor = "test"; + ReflectionTestUtils.setField(cmd, "hypervisor", hypervisor); + assertEquals(hypervisor, cmd.getHypervisor()); + } + + @Test + public void testGetId() { + Long id = 1234L; + ReflectionTestUtils.setField(cmd, "id", id); + assertEquals(id, cmd.getId()); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java index cdcb3c0445a..4fdd6d48896 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/CreateEntityDownloadURLAnswer.java @@ -23,18 +23,15 @@ import com.cloud.agent.api.Answer; public class CreateEntityDownloadURLAnswer extends Answer { - String resultString; - short resultCode; public static final short RESULT_SUCCESS = 1; public static final short RESULT_FAILURE = 0; public CreateEntityDownloadURLAnswer(String resultString, short resultCode) { super(); - this.resultString = resultString; - this.resultCode = resultCode; + this.details = resultString; + this.result = resultCode == RESULT_SUCCESS; } public CreateEntityDownloadURLAnswer() { } - } diff --git a/core/src/main/java/com/cloud/resource/ServerResourceBase.java b/core/src/main/java/com/cloud/resource/ServerResourceBase.java index f108d375b12..18121e21e51 100644 --- a/core/src/main/java/com/cloud/resource/ServerResourceBase.java +++ b/core/src/main/java/com/cloud/resource/ServerResourceBase.java @@ -19,6 +19,7 @@ package com.cloud.resource; +import java.io.File; import java.io.PrintWriter; import java.io.StringWriter; import java.net.NetworkInterface; @@ -33,6 +34,7 @@ import java.util.Map; import javax.naming.ConfigurationException; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; @@ -150,6 +152,39 @@ public abstract class ServerResourceBase implements ServerResource { return true; } + protected Answer listFilesAtPath(String nfsMountPoint, String relativePath, int startIndex, int pageSize) { + int count = 0; + File file = new File(nfsMountPoint, relativePath); + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List absPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + if (file.isFile()) { + count = 1; + names.add(file.getName()); + paths.add(file.getPath().replace(nfsMountPoint, "")); + absPaths.add(file.getPath()); + isDirs.add(file.isDirectory()); + sizes.add(file.length()); + modifiedList.add(file.lastModified()); + } else if (file.isDirectory()) { + String[] files = file.list(); + count = files.length; + for (int i = startIndex; i < startIndex + pageSize && i < count; i++) { + File f = new File(nfsMountPoint, relativePath + '/' + files[i]); + names.add(f.getName()); + paths.add(f.getPath().replace(nfsMountPoint, "")); + absPaths.add(f.getPath()); + isDirs.add(f.isDirectory()); + sizes.add(f.length()); + modifiedList.add(f.lastModified()); + } + } + return new ListDataStoreObjectsAnswer(file.exists(), count, names, paths, absPaths, isDirs, sizes, modifiedList); + } + protected void fillNetworkInformation(final StartupCommand cmd) { String[] info = null; if (privateNic != null) { diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java b/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java new file mode 100644 index 00000000000..eb8d0991c5c --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswer.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.command.browser; + +import com.cloud.agent.api.Answer; + +import java.util.Collections; +import java.util.List; + +public class ListDataStoreObjectsAnswer extends Answer { + + private boolean pathExists; + + private int count; + + private List names; + + private List paths; + + private List absPaths; + + private List isDirs; + + private List sizes; + + private List lastModified; + + public ListDataStoreObjectsAnswer() { + super(); + } + + public ListDataStoreObjectsAnswer(boolean pathExists, int count, List names, List paths, + List absPaths, List isDirs, + List sizes, + List lastModified) { + super(); + this.pathExists = pathExists; + this.count = count; + this.names = names; + this.paths = paths; + this.absPaths = absPaths; + this.isDirs = isDirs; + this.sizes = sizes; + this.lastModified = lastModified; + } + + public boolean isPathExists() { + return pathExists; + } + + public int getCount() { + return count; + } + + public List getNames() { + if (names == null) { + return Collections.emptyList(); + } + return names; + } + + public List getPaths() { + if (paths == null) { + return Collections.emptyList(); + } + return paths; + } + + public List getAbsPaths() { + if (absPaths == null) { + return Collections.emptyList(); + } + return absPaths; + } + + public List getIsDirs() { + if (isDirs == null) { + return Collections.emptyList(); + } + return isDirs; + } + + public List getSizes() { + if (sizes == null) { + return Collections.emptyList(); + } + return sizes; + } + + public List getLastModified() { + if (lastModified == null) { + return Collections.emptyList(); + } + return lastModified; + } +} diff --git a/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsCommand.java b/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsCommand.java new file mode 100644 index 00000000000..c5c0dc508ba --- /dev/null +++ b/core/src/main/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsCommand.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.command.browser; + +import com.cloud.agent.api.storage.StorageCommand; +import com.cloud.agent.api.to.DataStoreTO; + +public class ListDataStoreObjectsCommand extends StorageCommand { + + private DataStoreTO store; + + private String path; + + private int startIndex; + + private int pageSize; + + public ListDataStoreObjectsCommand() { + } + + public ListDataStoreObjectsCommand(DataStoreTO store, String path, int startIndex, int pageSize) { + super(); + this.store = store; + this.path = path; + this.startIndex = startIndex; + this.pageSize = pageSize; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public String getPath() { + return path; + } + + public DataStoreTO getStore() { + return store; + } + + public int getStartIndex() { + return startIndex; + } + + public int getPageSize() { + return pageSize; + } +} diff --git a/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java b/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java index 5f3323d1df7..ed64e1482a6 100644 --- a/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java +++ b/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java @@ -17,6 +17,8 @@ package com.cloud.resource; import com.cloud.utils.net.NetUtils; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.commons.collections.CollectionUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -28,6 +30,8 @@ import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import javax.naming.ConfigurationException; +import java.io.File; +import java.io.IOException; import java.net.NetworkInterface; import java.net.SocketException; import java.util.ArrayList; @@ -234,4 +238,37 @@ public class ServerResourceBaseTest { Assert.assertEquals(networkInterfaceMock3, serverResourceBaseSpy.storageNic); Assert.assertEquals(networkInterfaceMock4, serverResourceBaseSpy.storageNic2); } + + @Test + public void testListFilesAtPath() throws IOException { + String nfsMountPoint = "/tmp/nfs"; + String relativePath = "test"; + int startIndex = 0; + int pageSize = 10; + + // create a test directory with some files + File testDir = new File(nfsMountPoint, relativePath); + testDir.mkdirs(); + File file1 = new File(testDir, "file1.txt"); + File file2 = new File(testDir, "file2.txt"); + file1.createNewFile(); + file2.createNewFile(); + + ListDataStoreObjectsAnswer result = (ListDataStoreObjectsAnswer) serverResourceBaseSpy.listFilesAtPath(nfsMountPoint, relativePath, startIndex, pageSize); + + Assert.assertTrue(result.getResult()); + Assert.assertEquals(2, result.getCount()); + List expectedNames = Arrays.asList("file1.txt", "file2.txt"); + List expectedPaths = Arrays.asList("/test/file1.txt", "/test/file2.txt"); + List expectedAbsPaths = Arrays.asList(nfsMountPoint + "/test/file1.txt", nfsMountPoint + "/test/file2.txt"); + List expectedIsDirs = Arrays.asList(false, false); + List expectedSizes = Arrays.asList(file2.length(), file1.length()); + List expectedModifiedList = Arrays.asList(file2.lastModified(), file1.lastModified()); + Assert.assertTrue(CollectionUtils.isEqualCollection(expectedNames, result.getNames())); + Assert.assertTrue(CollectionUtils.isEqualCollection(expectedPaths, result.getPaths())); + Assert.assertTrue(CollectionUtils.isEqualCollection(expectedAbsPaths, result.getAbsPaths())); + Assert.assertTrue(CollectionUtils.isEqualCollection(expectedIsDirs, result.getIsDirs())); + Assert.assertTrue(CollectionUtils.isEqualCollection(expectedSizes, result.getSizes())); + Assert.assertTrue(CollectionUtils.isEqualCollection(expectedModifiedList, result.getLastModified())); + } } diff --git a/core/src/test/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswerTest.java b/core/src/test/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswerTest.java new file mode 100644 index 00000000000..fdf3c5beb7e --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsAnswerTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.command.browser; + + +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ListDataStoreObjectsAnswerTest { + + @Test + public void testGetters() { + ListDataStoreObjectsAnswer answer = new ListDataStoreObjectsAnswer(true, 2, + Arrays.asList("file1", "file2"), Arrays.asList("path1", "path2"), + Arrays.asList("/mnt/datastore/path1", "/mnt/datastore/path2"), Arrays.asList(false, false), + Arrays.asList(1024L, 2048L), Arrays.asList(123456789L, 987654321L)); + + assertTrue(answer.isPathExists()); + assertEquals(2, answer.getCount()); + assertEquals(Arrays.asList("file1", "file2"), answer.getNames()); + assertEquals(Arrays.asList("path1", "path2"), answer.getPaths()); + assertEquals(Arrays.asList("/mnt/datastore/path1", "/mnt/datastore/path2"), answer.getAbsPaths()); + assertEquals(Arrays.asList(false, false), answer.getIsDirs()); + assertEquals(Arrays.asList(1024L, 2048L), answer.getSizes()); + assertEquals(Arrays.asList(123456789L, 987654321L), answer.getLastModified()); + } + + @Test + public void testEmptyLists() { + ListDataStoreObjectsAnswer answer = new ListDataStoreObjectsAnswer(true, 0, null, null, null, null, null, null); + + assertTrue(answer.isPathExists()); + assertEquals(0, answer.getCount()); + assertEquals(Collections.emptyList(), answer.getNames()); + assertEquals(Collections.emptyList(), answer.getPaths()); + assertEquals(Collections.emptyList(), answer.getAbsPaths()); + assertEquals(Collections.emptyList(), answer.getIsDirs()); + assertEquals(Collections.emptyList(), answer.getSizes()); + assertEquals(Collections.emptyList(), answer.getLastModified()); + } +} diff --git a/core/src/test/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsCommandTest.java b/core/src/test/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsCommandTest.java new file mode 100644 index 00000000000..b7c037a2bef --- /dev/null +++ b/core/src/test/java/org/apache/cloudstack/storage/command/browser/ListDataStoreObjectsCommandTest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.command.browser; + +import com.cloud.agent.api.to.DataStoreTO; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class ListDataStoreObjectsCommandTest { + + @Test + public void testStartIndex() { + DataStoreTO dataStore = Mockito.mock(DataStoreTO.class); + ListDataStoreObjectsCommand cmd = new ListDataStoreObjectsCommand(dataStore, "path", 40, 10); + assertEquals(40, cmd.getStartIndex()); + assertEquals(10, cmd.getPageSize()); + assertEquals("path", cmd.getPath()); + assertEquals(dataStore, cmd.getStore()); + assertFalse(cmd.executeInSequence()); + } + +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java index 7bf845d3ec5..481d0ebbc76 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/StorageOrchestrationService.java @@ -24,4 +24,6 @@ import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; public interface StorageOrchestrationService { MigrationResponse migrateData(Long srcDataStoreId, List destDatastores, MigrationPolicy migrationPolicy); + + MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List templateIdList, List snapshotIdList); } diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java index ea6318f0591..0a761cb7fbb 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java @@ -91,20 +91,17 @@ public class DataMigrationUtility { * "Ready" "Allocated", "Destroying", "Destroyed", "Failed". If this is the case, and if the migration policy is complete, * the migration is terminated. */ - private boolean filesReadyToMigrate(Long srcDataStoreId) { + public boolean filesReadyToMigrate(Long srcDataStoreId, List templates, List snapshots, List volumes) { State[] validStates = {State.Ready, State.Allocated, State.Destroying, State.Destroyed, State.Failed}; boolean isReady = true; - List templates = templateDataStoreDao.listByStoreId(srcDataStoreId); for (TemplateDataStoreVO template : templates) { isReady &= (Arrays.asList(validStates).contains(template.getState())); LOGGER.trace(String.format("template state: %s", template.getState())); } - List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStoreId, DataStoreRole.Image); for (SnapshotDataStoreVO snapshot : snapshots) { isReady &= (Arrays.asList(validStates).contains(snapshot.getState())); LOGGER.trace(String.format("snapshot state: %s", snapshot.getState())); } - List volumes = volumeDataStoreDao.listByStoreId(srcDataStoreId); for (VolumeDataStoreVO volume : volumes) { isReady &= (Arrays.asList(validStates).contains(volume.getState())); LOGGER.trace(String.format("volume state: %s", volume.getState())); @@ -112,6 +109,13 @@ public class DataMigrationUtility { return isReady; } + private boolean filesReadyToMigrate(Long srcDataStoreId) { + List templates = templateDataStoreDao.listByStoreId(srcDataStoreId); + List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStoreId, DataStoreRole.Image); + List volumes = volumeDataStoreDao.listByStoreId(srcDataStoreId); + return filesReadyToMigrate(srcDataStoreId, templates, snapshots, volumes); + } + protected void checkIfCompleteMigrationPossible(ImageStoreService.MigrationPolicy policy, Long srcDataStoreId) { if (policy == ImageStoreService.MigrationPolicy.COMPLETE) { if (!filesReadyToMigrate(srcDataStoreId)) { @@ -158,7 +162,19 @@ public class DataMigrationUtility { } protected List getSortedValidSourcesList(DataStore srcDataStore, Map, Long>> snapshotChains, - Map, Long>> childTemplates) { + Map, Long>> childTemplates, List templates, List snapshots) { + List files = new ArrayList<>(); + + files.addAll(getAllReadyTemplates(srcDataStore, childTemplates, templates)); + files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshots)); + + files = sortFilesOnSize(files, snapshotChains); + + return files; + } + + protected List getSortedValidSourcesList(DataStore srcDataStore, Map, Long>> snapshotChains, + Map, Long>> childTemplates) { List files = new ArrayList<>(); files.addAll(getAllReadyTemplates(srcDataStore, childTemplates)); files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains)); @@ -187,10 +203,8 @@ public class DataMigrationUtility { return files; } - protected List getAllReadyTemplates(DataStore srcDataStore, Map, Long>> childTemplates) { - + protected List getAllReadyTemplates(DataStore srcDataStore, Map, Long>> childTemplates, List templates) { List files = new LinkedList<>(); - List templates = templateDataStoreDao.listByStoreId(srcDataStore.getId()); for (TemplateDataStoreVO template : templates) { VMTemplateVO templateVO = templateDao.findById(template.getTemplateId()); if (template.getState() == ObjectInDataStoreStateMachine.State.Ready && templateVO != null && @@ -211,13 +225,17 @@ public class DataMigrationUtility { return (List) (List) files; } + protected List getAllReadyTemplates(DataStore srcDataStore, Map, Long>> childTemplates) { + List templates = templateDataStoreDao.listByStoreId(srcDataStore.getId()); + return getAllReadyTemplates(srcDataStore, childTemplates, templates); + } + /** Returns parent snapshots and snapshots that do not have any children; snapshotChains comprises of the snapshot chain info * for each parent snapshot and the cumulative size of the chain - this is done to ensure that all the snapshots in a chain * are migrated to the same datastore */ - protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains) { + protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains, List snapshots) { List files = new LinkedList<>(); - List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStore.getId(), DataStoreRole.Image); for (SnapshotDataStoreVO snapshot : snapshots) { SnapshotVO snapshotVO = snapshotDao.findById(snapshot.getSnapshotId()); if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && @@ -246,6 +264,11 @@ public class DataMigrationUtility { return (List) (List) files; } + protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains) { + List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStore.getId(), DataStoreRole.Image); + return getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshots); + } + protected Long getTotalChainSize(List chain) { Long size = 0L; for (DataObject dataObject : chain) { @@ -254,9 +277,8 @@ public class DataMigrationUtility { return size; } - protected List getAllReadyVolumes(DataStore srcDataStore) { + protected List getAllReadyVolumes(DataStore srcDataStore, List volumes) { List files = new LinkedList<>(); - List volumes = volumeDataStoreDao.listByStoreId(srcDataStore.getId()); for (VolumeDataStoreVO volume : volumes) { if (volume.getState() == ObjectInDataStoreStateMachine.State.Ready) { VolumeInfo volumeInfo = volumeFactory.getVolume(volume.getVolumeId(), srcDataStore); @@ -268,6 +290,11 @@ public class DataMigrationUtility { return files; } + protected List getAllReadyVolumes(DataStore srcDataStore) { + List volumes = volumeDataStoreDao.listByStoreId(srcDataStore.getId()); + return getAllReadyVolumes(srcDataStore, volumes); + } + /** Returns the count of active SSVMs - SSVM with agents in connected state, so as to dynamically increase the thread pool * size when SSVMs scale */ diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index eef0cdef2fe..873ddb5d80b 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.engine.orchestration; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Hashtable; @@ -30,6 +31,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -52,7 +54,9 @@ import org.apache.cloudstack.storage.ImageStoreService.MigrationPolicy; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.math3.stat.descriptive.moment.Mean; import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; import org.apache.log4j.Logger; @@ -219,6 +223,82 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra return handleResponse(futures, migrationPolicy, message, success); } + @Override + public MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreId, List templateIdList, + List snapshotIdList) { + List files; + boolean success = true; + String message = null; + + DataStore srcDatastore = dataStoreManager.getDataStore(srcImgStoreId, DataStoreRole.Image); + Map, Long>> snapshotChains = new HashMap<>(); + Map, Long>> childTemplates = new HashMap<>(); + + List templates = templateDataStoreDao.listByStoreIdAndTemplateIds(srcImgStoreId, templateIdList); + List snapshots = snapshotDataStoreDao.listByStoreAndSnapshotIds(srcImgStoreId, DataStoreRole.Image, snapshotIdList); + + if (!migrationHelper.filesReadyToMigrate(srcImgStoreId, templates, snapshots, Collections.emptyList())) { + throw new CloudRuntimeException("Migration failed as there are data objects which are not Ready - i.e, they may be in Migrating, creating, copying, etc. states"); + } + files = migrationHelper.getSortedValidSourcesList(srcDatastore, snapshotChains, childTemplates, templates, snapshots); + + if (files.isEmpty()) { + return new MigrationResponse(String.format("No files in Image store: %s to migrate", srcDatastore.getUuid()), null, true); + } + + Map> storageCapacities = new Hashtable<>(); + storageCapacities.put(srcImgStoreId, new Pair<>(null, null)); + storageCapacities.put(destImgStoreId, new Pair<>(null, null)); + storageCapacities = getStorageCapacities(storageCapacities, srcImgStoreId); + storageCapacities = getStorageCapacities(storageCapacities, destImgStoreId); + + ThreadPoolExecutor executor = new ThreadPoolExecutor(numConcurrentCopyTasksPerSSVM, numConcurrentCopyTasksPerSSVM, 30, + TimeUnit.MINUTES, new MigrateBlockingQueue<>(numConcurrentCopyTasksPerSSVM)); + List>> futures = new ArrayList<>(); + Date start = new Date(); + + while (true) { + DataObject chosenFileForMigration = null; + if (!files.isEmpty()) { + chosenFileForMigration = files.remove(0); + } + + if (chosenFileForMigration == null) { + message = "Migration completed"; + break; + } + + if (chosenFileForMigration.getPhysicalSize() > storageCapacities.get(destImgStoreId).first()) { + s_logger.debug(String.format("%s: %s too large to be migrated to %s", chosenFileForMigration.getType().name(), chosenFileForMigration.getUuid(), destImgStoreId)); + continue; + } + + if (storageCapacityBelowThreshold(storageCapacities, destImgStoreId)) { + storageCapacities = migrateAway(chosenFileForMigration, storageCapacities, snapshotChains, childTemplates, srcDatastore, destImgStoreId, executor, futures); + } else { + message = "Migration failed. Destination store doesn't have enough capacity for migration"; + success = false; + break; + } + } + Date end = new Date(); + + // Migrate any new child snapshots if created during migration + List migratedSnapshotIdList = snapshotChains.keySet().stream().map(DataObject::getId).collect(Collectors.toList()); + if (!CollectionUtils.isEmpty(migratedSnapshotIdList)) { + List snaps = snapshotDataStoreDao.findSnapshots(srcImgStoreId, start, end); + snaps.forEach(snap -> { + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snap.getSnapshotId(), snap.getDataStoreId(), DataStoreRole.Image); + SnapshotInfo parentSnapshot = snapshotInfo.getParent(); + if (snapshotInfo.getDataStore().getId() == srcImgStoreId && parentSnapshot != null && migratedSnapshotIdList.contains(parentSnapshot.getSnapshotId())) { + futures.add(executor.submit(new MigrateDataTask(snapshotInfo, srcDatastore, dataStoreManager.getDataStore(destImgStoreId, DataStoreRole.Image)))); + } + }); + } + + return handleResponse(futures, null, message, success); + } + protected Pair migrateCompleted(Long destDatastoreId, DataStore srcDatastore, List files, MigrationPolicy migrationPolicy, int skipped) { String message = ""; boolean success = true; @@ -295,7 +375,8 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra } } message += ". successful migrations: "+successCount; - return new MigrationResponse(message, migrationPolicy.toString(), success); + String policy = migrationPolicy != null ? migrationPolicy.toString() : null; + return new MigrationResponse(message, policy, success); } private void handleSnapshotMigration(Long srcDataStoreId, Date start, Date end, MigrationPolicy policy, diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java index b1d7f21b4f0..708a77a8f9e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDao.java @@ -88,4 +88,6 @@ public interface VMTemplateDao extends GenericDao, StateDao< VMTemplateVO findLatestTemplateByName(String name); List findTemplatesLinkedToUserdata(long userdataId); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java index 08b98f9869e..031bcb3af7b 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplateDaoImpl.java @@ -17,6 +17,7 @@ package com.cloud.storage.dao; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -26,6 +27,7 @@ import javax.naming.ConfigurationException; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -99,6 +101,7 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem private SearchBuilder InactiveUnremovedTmpltSearch; private SearchBuilder LatestTemplateByHypervisorTypeSearch; private SearchBuilder userDataSearch; + private SearchBuilder templateIdSearch; @Inject ResourceTagDao _tagsDao; @@ -427,6 +430,11 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem userDataSearch.and("state", userDataSearch.entity().getState(), SearchCriteria.Op.EQ); userDataSearch.done(); + + templateIdSearch = createSearchBuilder(); + templateIdSearch.and("idIN", templateIdSearch.entity().getId(), SearchCriteria.Op.IN); + templateIdSearch.done(); + return result; } @@ -648,6 +656,16 @@ public class VMTemplateDaoImpl extends GenericDaoBase implem return listBy(sc); } + @Override + public List listByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return Collections.emptyList(); + } + SearchCriteria sc = templateIdSearch.create(); + sc.setParameters("idIN", ids.toArray()); + return listBy(sc, null); + } + @Override @DB public boolean remove(Long id) { diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java index d00eeceec7c..a3ce03a74c3 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDao.java @@ -52,4 +52,8 @@ public interface VMTemplatePoolDao extends GenericDao listByTemplatePath(String templatePath); + + List listByPoolIdAndInstallPath(Long poolId, List pathList); + + List listByTemplateId(long templateId, List poolIds); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java index 479e02eb5ba..d938bebb18e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VMTemplatePoolDaoImpl.java @@ -19,6 +19,7 @@ package com.cloud.storage.dao; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; @@ -28,6 +29,7 @@ import javax.inject.Inject; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -86,7 +88,7 @@ public class VMTemplatePoolDaoImpl extends GenericDaoBase listByPoolIdAndInstallPath(Long poolId, List pathList) { + if (CollectionUtils.isEmpty(pathList)) { + return Collections.emptyList(); + } + SearchCriteria sc = templatePathSearch.create(); + sc.setParameters("pool_id", poolId); + sc.setParameters("install_path", pathList.toArray()); + return listBy(sc); + } + + @Override + public List listByTemplateId(long templateId, List poolIds) { + if (CollectionUtils.isEmpty(poolIds)) { + return Collections.emptyList(); + } + SearchCriteria sc = PoolTemplateSearch.create(); + sc.setParameters("template_id", templateId); + sc.setParameters("pool_id", poolIds.toArray()); + return listBy(sc); + } + @Override public boolean updateState(State currentState, Event event, State nextState, DataObjectInStore vo, Object data) { VMTemplateStoragePoolVO templatePool = (VMTemplateStoragePoolVO)vo; diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java index 3cdaa3b05ad..79899b7119e 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDao.java @@ -149,4 +149,8 @@ public interface VolumeDao extends GenericDao, StateDao listByPoolIdAndPaths(long id, List pathList); + + List listByIds(List ids); } diff --git a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java index 2b5e34c6876..056b7206d72 100644 --- a/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/storage/dao/VolumeDaoImpl.java @@ -20,11 +20,13 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import javax.inject.Inject; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -63,6 +65,8 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol protected final SearchBuilder AllFieldsSearch; protected final SearchBuilder diskOfferingSearch; protected final SearchBuilder RootDiskStateSearch; + private final SearchBuilder storeAndInstallPathSearch; + private final SearchBuilder volumeIdSearch; protected GenericSearchBuilder CountByAccount; protected GenericSearchBuilder primaryStorageSearch; protected GenericSearchBuilder primaryStorageSearch2; @@ -473,6 +477,16 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol secondaryStorageSearch.and("states", secondaryStorageSearch.entity().getState(), Op.NIN); secondaryStorageSearch.and("isRemoved", secondaryStorageSearch.entity().getRemoved(), Op.NULL); secondaryStorageSearch.done(); + + storeAndInstallPathSearch = createSearchBuilder(); + storeAndInstallPathSearch.and("poolId", storeAndInstallPathSearch.entity().getPoolId(), Op.EQ); + storeAndInstallPathSearch.and("pathIN", storeAndInstallPathSearch.entity().getPath(), Op.IN); + storeAndInstallPathSearch.done(); + + volumeIdSearch = createSearchBuilder(); + volumeIdSearch.and("idIN", volumeIdSearch.entity().getId(), Op.IN); + volumeIdSearch.done(); + } @Override @@ -775,4 +789,26 @@ public class VolumeDaoImpl extends GenericDaoBase implements Vol remove(volume.getId()); } } + + @Override + public List listByPoolIdAndPaths(long id, List pathList) { + if (CollectionUtils.isEmpty(pathList)) { + return Collections.emptyList(); + } + + SearchCriteria sc = storeAndInstallPathSearch.create(); + sc.setParameters("poolId", id); + sc.setParameters("pathIN", pathList.toArray()); + return listBy(sc); + } + + @Override + public List listByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return Collections.emptyList(); + } + SearchCriteria sc = volumeIdSearch.create(); + sc.setParameters("idIN", ids.toArray()); + return listBy(sc, null); + } } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/BasicTemplateDataStoreDaoImpl.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/BasicTemplateDataStoreDaoImpl.java index 3ea63d059a6..431686f784d 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/BasicTemplateDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/BasicTemplateDataStoreDaoImpl.java @@ -229,6 +229,16 @@ public class BasicTemplateDataStoreDaoImpl extends GenericDaoBase listByStoreIdAndInstallPaths(long storeId, List installPaths) { + return null; + } + + @Override + public List listByStoreIdAndTemplateIds(long storeId, List templateIds) { + return null; + } + @Override public boolean updateState(ObjectInDataStoreStateMachine.State currentState, ObjectInDataStoreStateMachine.Event event, ObjectInDataStoreStateMachine.State nextState, DataObjectInStore vo, Object data) { return false; diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadDao.java new file mode 100644 index 00000000000..964e729c8aa --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadDao.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; + +import java.util.Date; +import java.util.List; + +public interface ImageStoreObjectDownloadDao extends GenericDao { + ImageStoreObjectDownloadVO findByStoreIdAndPath(long storeId, String path); + + List listToExpire(Date date); +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadDaoImpl.java new file mode 100644 index 00000000000..918ab8f43b7 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadDaoImpl.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; +import org.springframework.stereotype.Component; + +import javax.naming.ConfigurationException; +import java.util.Date; +import java.util.List; +import java.util.Map; + +@Component +public class ImageStoreObjectDownloadDaoImpl extends GenericDaoBase implements ImageStoreObjectDownloadDao { + private SearchBuilder storeIdPathSearch; + + private SearchBuilder createdSearch; + + public ImageStoreObjectDownloadDaoImpl() { + } + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + storeIdPathSearch = createSearchBuilder(); + storeIdPathSearch.and("store_id", storeIdPathSearch.entity().getStoreId(), SearchCriteria.Op.EQ); + storeIdPathSearch.and("path", storeIdPathSearch.entity().getPath(), SearchCriteria.Op.EQ); + storeIdPathSearch.done(); + + createdSearch = createSearchBuilder(); + createdSearch.and("created", createdSearch.entity().getCreated(), SearchCriteria.Op.LTEQ); + createdSearch.done(); + + return true; + } + + @Override + public ImageStoreObjectDownloadVO findByStoreIdAndPath(long storeId, String path) { + SearchCriteria sc = storeIdPathSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("path", path); + return findOneBy(sc); + } + + @Override + public List listToExpire(Date date) { + SearchCriteria sc = createdSearch.create(); + sc.setParameters("created", date); + return listBy(sc); + } +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadVO.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadVO.java new file mode 100644 index 00000000000..a698184c0e7 --- /dev/null +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/ImageStoreObjectDownloadVO.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.datastore.db; + +import com.cloud.utils.db.GenericDao; +import org.apache.cloudstack.storage.ImageStoreObjectDownload; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.Date; + +@Entity +@Table(name = "image_store_object_download") +public class ImageStoreObjectDownloadVO implements ImageStoreObjectDownload { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id", nullable = false) + private long id; + + @Column(name = "store_id", nullable = false) + private Long storeId; + + @Column(name = "path", nullable = false) + private String path; + + @Column(name = "download_url", nullable = false) + private String downloadUrl; + + @Column(name = GenericDao.CREATED_COLUMN) + private Date created; + + public ImageStoreObjectDownloadVO() { + } + + public ImageStoreObjectDownloadVO(Long storeId, String path, String downloadUrl) { + this.storeId = storeId; + this.path = path; + this.downloadUrl = downloadUrl; + } + + public long getId() { + return id; + } + + public Long getStoreId() { + return storeId; + } + + public String getPath() { + return path; + } + + public String getDownloadUrl() { + return downloadUrl; + } + + public Date getCreated() { + return created; + } + +} diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java index 1ddde246f79..344ff8b2a69 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDao.java @@ -97,6 +97,10 @@ StateDao listReadyByVolumeId(long volumeId); + List listByStoreAndInstallPaths(long storeId, DataStoreRole role, List pathList); + + List listByStoreAndSnapshotIds(long storeId, DataStoreRole role, List snapshotIds); + List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status); SnapshotDataStoreVO findOneBySnapshotAndDatastoreRole(long snapshotId, DataStoreRole role); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java index 657551ae8b7..98cb6ca5b42 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/SnapshotDataStoreDaoImpl.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.storage.datastore.db; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -65,6 +66,8 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase idStateNeqSearch; protected SearchBuilder snapshotVOSearch; private SearchBuilder snapshotCreatedSearch; + private SearchBuilder dataStoreAndInstallPathSearch; + private SearchBuilder storeAndSnapshotIdsSearch; private SearchBuilder storeSnapshotDownloadStatusSearch; protected static final List HYPERVISORS_SUPPORTING_SNAPSHOTS_CHAINING = List.of(Hypervisor.HypervisorType.XenServer); @@ -132,6 +135,18 @@ public class SnapshotDataStoreDaoImpl extends GenericDaoBase listByStoreAndInstallPaths(long storeId, DataStoreRole role, List pathList) { + if (CollectionUtils.isEmpty(pathList)) { + return Collections.emptyList(); + } + + SearchCriteria sc = dataStoreAndInstallPathSearch.create(); + sc.setParameters(STORE_ID, storeId); + sc.setParameters(STORE_ROLE, role); + sc.setParameters("install_pathIN", pathList.toArray()); + return listBy(sc); + } + + @Override + public List listByStoreAndSnapshotIds(long storeId, DataStoreRole role, List snapshotIds) { + if (CollectionUtils.isEmpty(snapshotIds)) { + return Collections.emptyList(); + } + + SearchCriteria sc = storeAndSnapshotIdsSearch.create(); + sc.setParameters(STORE_ID, storeId); + sc.setParameters(STORE_ROLE, role); + sc.setParameters("snapshot_idIN", snapshotIds.toArray()); + return listBy(sc); + } + @Override public List listBySnasphotStoreDownloadStatus(long snapshotId, long storeId, VMTemplateStorageResourceAssoc.Status... status) { SearchCriteria sc = storeSnapshotDownloadStatusSearch.create(); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java index f8e210ac326..441649ec5c6 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/TemplateDataStoreDao.java @@ -93,4 +93,8 @@ public interface TemplateDataStoreDao extends GenericDao listTemplateDownloadUrlsByStoreId(long storeId); + + List listByStoreIdAndInstallPaths(long storeId, List installPaths); + + List listByStoreIdAndTemplateIds(long storeId, List templateIds); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java index b3b2ece9043..c3a4b58fbd5 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/VolumeDataStoreDao.java @@ -57,4 +57,6 @@ public interface VolumeDataStoreDao extends GenericDao, List listVolumeDownloadUrlsByZoneId(long zoneId); List listByVolume(long volumeId, long storeId); + + List listByStoreIdAndInstallPaths(Long storeId, List paths); } diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml index fdf471d4ba4..0c46c5ff934 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml @@ -45,6 +45,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql index 60b200c6613..dd730058b7b 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41810to41900.sql @@ -181,6 +181,26 @@ CREATE TABLE `cloud`.`vm_scheduled_job` ( ALTER TABLE `cloud`.`kubernetes_cluster` ADD COLUMN `cluster_type` varchar(64) DEFAULT 'CloudManaged' COMMENT 'type of cluster'; ALTER TABLE `cloud`.`kubernetes_cluster` MODIFY COLUMN `kubernetes_version_id` bigint unsigned NULL COMMENT 'the ID of the Kubernetes version of this Kubernetes cluster'; +-- Add indexes for data store browser +ALTER TABLE `cloud`.`template_spool_ref` ADD INDEX `i_template_spool_ref__install_path`(`install_path`); +ALTER TABLE `cloud`.`volumes` ADD INDEX `i_volumes__path`(`path`); +ALTER TABLE `cloud`.`snapshot_store_ref` ADD INDEX `i_snapshot_store_ref__install_path`(`install_path`); +ALTER TABLE `cloud`.`template_store_ref` ADD INDEX `i_template_store_ref__install_path`(`install_path`); + +-- Add table for image store object download +DROP TABLE IF EXISTS `cloud`.`image_store_object_download`; +CREATE TABLE `cloud`.`image_store_object_download` ( + `id` bigint unsigned NOT NULL auto_increment COMMENT 'id', + `store_id` bigint unsigned NOT NULL COMMENT 'image store id', + `path` varchar(255) NOT NULL COMMENT 'path on store', + `download_url` varchar(255) NOT NULL COMMENT 'download url', + `created` datetime COMMENT 'date created', + PRIMARY KEY (`id`), + UNIQUE KEY (`store_id`, `path`), + INDEX `i_image_store_object_download__created`(`created`), + CONSTRAINT `fk_image_store_object_download__store_id` FOREIGN KEY (`store_id`) REFERENCES `image_store`(`id`) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8; + -- Set removed state for all removed accounts UPDATE `cloud`.`account` SET state='removed' WHERE `removed` IS NOT NULL; diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java index ef3d20a8ea0..f9684d648c2 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/TemplateDataStoreDaoImpl.java @@ -70,6 +70,8 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase downloadTemplateSearch; private SearchBuilder uploadTemplateStateSearch; private SearchBuilder directDownloadTemplateSeach; + private SearchBuilder imageStoreAndInstallPathSearch; + private SearchBuilder storeIdAndTemplateIdsSearch; private SearchBuilder templateOnlySearch; private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update template_store_ref set download_url_created=? where download_url_created is not null and store_id in (select id from image_store where data_center_id=?)"; @@ -163,6 +165,16 @@ public class TemplateDataStoreDaoImpl extends GenericDaoBase listByStoreIdAndInstallPaths(long storeId, List installPaths) { + if (CollectionUtils.isEmpty(installPaths)) { + return Collections.emptyList(); + } + + SearchCriteria sc = imageStoreAndInstallPathSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("install_pathIN", installPaths.toArray()); + return listBy(sc); + } + + @Override + public List listByStoreIdAndTemplateIds(long storeId, List templateIds) { + if (CollectionUtils.isEmpty(templateIds)) { + return Collections.emptyList(); + } + SearchCriteria sc = storeIdAndTemplateIdsSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("template_idIN", templateIds.toArray()); + return listBy(sc); + } + @Override public void expireDnldUrlsForZone(Long dcId){ TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java index dca2e9a862e..dcdc9ea56c2 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/image/db/VolumeDataStoreDaoImpl.java @@ -18,6 +18,7 @@ package org.apache.cloudstack.storage.image.db; import java.sql.PreparedStatement; import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.List; import java.util.Map; @@ -27,6 +28,7 @@ import javax.naming.ConfigurationException; import com.cloud.utils.db.Filter; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.collections.CollectionUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import org.apache.cloudstack.engine.subsystem.api.storage.DataObjectInStore; @@ -60,12 +62,13 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase uploadVolumeSearch; private SearchBuilder volumeOnlySearch; private SearchBuilder uploadVolumeStateSearch; + private SearchBuilder imageStoreAndInstallPathSearch; private static final String EXPIRE_DOWNLOAD_URLS_FOR_ZONE = "update volume_store_ref set download_url_created=? where download_url_created is not null and store_id in (select id from image_store where data_center_id=?)"; @Inject DataStoreManager storeMgr; @Inject - VolumeDao volumeDao; + VolumeDao volumeDao;; @Override public boolean configure(String name, Map params) throws ConfigurationException { @@ -118,6 +121,11 @@ public class VolumeDataStoreDaoImpl extends GenericDaoBase listByStoreIdAndInstallPaths(Long storeId, List paths) { + if (CollectionUtils.isEmpty(paths)) { + return Collections.emptyList(); + } + + SearchCriteria sc = imageStoreAndInstallPathSearch.create(); + sc.setParameters("store_id", storeId); + sc.setParameters("install_pathIN", paths.toArray()); + return listBy(sc); + } + @Override public List listUploadedVolumesByStoreId(long id) { SearchCriteria sc = uploadVolumeSearch.create(); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 6bbafcacef2..0019f1de937 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -50,6 +50,7 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.cloudstack.api.ApiConstants.IoDriverPolicy; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -4679,6 +4680,12 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv return true; } + public Answer listFilesAtPath(ListDataStoreObjectsCommand command) { + DataStoreTO store = command.getStore(); + KVMStoragePool storagePool = storagePoolManager.getStoragePool(StoragePoolType.NetworkFilesystem, store.getUuid()); + return listFilesAtPath(storagePool.getLocalPath(), command.getPath(), command.getStartIndex(), command.getPageSize()); + } + public boolean addNetworkRules(final String vmName, final String vmId, final String guestIP, final String guestIP6, final String sig, final String seq, final String mac, final String rules, final String vif, final String brname, final String secIps) { if (!canBridgeFirewall) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListDataStoreObjectsCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListDataStoreObjectsCommandWrapper.java new file mode 100644 index 00000000000..f3544135509 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtListDataStoreObjectsCommandWrapper.java @@ -0,0 +1,35 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; + +@ResourceWrapper(handles = ListDataStoreObjectsCommand.class) +public final class LibvirtListDataStoreObjectsCommandWrapper extends CommandWrapper { + @Override + public Answer execute(final ListDataStoreObjectsCommand command, + final LibvirtComputingResource libvirtComputingResource) { + return libvirtComputingResource.listFilesAtPath(command); + } +} diff --git a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 79fa7807544..129e4bf343e 100644 --- a/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/main/java/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -48,9 +48,17 @@ import java.util.stream.Collectors; import javax.naming.ConfigurationException; import javax.xml.datatype.XMLGregorianCalendar; +import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; +import com.vmware.vim25.FileInfo; +import com.vmware.vim25.FileQueryFlags; +import com.vmware.vim25.FolderFileInfo; +import com.vmware.vim25.HostDatastoreBrowserSearchResults; +import com.vmware.vim25.HostDatastoreBrowserSearchSpec; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.storage.command.CopyCommand; import org.apache.cloudstack.storage.command.StorageSubSystemCommand; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @@ -615,6 +623,8 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes answer = execute((CheckGuestOsMappingCommand) cmd); } else if (clz == GetHypervisorGuestOsNamesCommand.class) { answer = execute((GetHypervisorGuestOsNamesCommand) cmd); + } else if (clz == ListDataStoreObjectsCommand.class) { + answer = execute((ListDataStoreObjectsCommand) cmd); } else { answer = Answer.createUnsupportedCommandAnswer(cmd); } @@ -7783,6 +7793,75 @@ public class VmwareResource extends ServerResourceBase implements StoragePoolRes } } + protected ListDataStoreObjectsAnswer execute(ListDataStoreObjectsCommand cmd) { + String path = cmd.getPath(); + int startIndex = cmd.getStartIndex(); + int pageSize = cmd.getPageSize(); + PrimaryDataStoreTO dataStore = (PrimaryDataStoreTO) cmd.getStore(); + + if (path.startsWith("/")) { + path = path.substring(1); + } + + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + VmwareContext context = getServiceContext(); + VmwareHypervisorHost hyperHost = getHyperHost(context); + ManagedObjectReference morDatastore = null; + + int count = 0; + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List absPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + + try { + morDatastore = HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(hyperHost, dataStore.getUuid()); + + DatastoreMO dsMo = new DatastoreMO(context, morDatastore); + HostDatastoreBrowserMO browserMo = dsMo.getHostDatastoreBrowserMO(); + FileQueryFlags fqf = new FileQueryFlags(); + fqf.setFileSize(true); + fqf.setFileType(true); + fqf.setModification(true); + fqf.setFileOwner(false); + + HostDatastoreBrowserSearchSpec spec = new HostDatastoreBrowserSearchSpec(); + spec.setSearchCaseInsensitive(true); + spec.setDetails(fqf); + + String dsPath = String.format("[%s] %s", dsMo.getName(), path); + + HostDatastoreBrowserSearchResults results = browserMo.searchDatastore(dsPath, spec); + List fileInfoList = results.getFile(); + count = fileInfoList.size(); + for (int i = startIndex; i < startIndex + pageSize && i < count; i++) { + FileInfo file = fileInfoList.get(i); + + names.add(file.getPath()); + paths.add(path + "/" + file.getPath()); + absPaths.add(dsPath + "/" + file.getPath()); + isDirs.add(file instanceof FolderFileInfo); + sizes.add(file.getFileSize()); + modifiedList.add(file.getModification().toGregorianCalendar().getTimeInMillis()); + } + + return new ListDataStoreObjectsAnswer(true, count, names, paths, absPaths, isDirs, sizes, modifiedList); + } catch (Exception e) { + if (e.getMessage().contains("was not found")) { + return new ListDataStoreObjectsAnswer(false, count, names, paths, absPaths, isDirs, sizes, modifiedList); + } + String errorMsg = String.format("Failed to list files at path [%s] due to: [%s].", path, e.getMessage()); + s_logger.error(errorMsg, e); + } + + return null; + } + protected GetHypervisorGuestOsNamesAnswer execute(GetHypervisorGuestOsNamesCommand cmd) { String keyword = cmd.getKeyword(); s_logger.info("Getting guest os names in the hypervisor"); diff --git a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java index da4273d0db6..58d8e5ef254 100644 --- a/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java +++ b/plugins/hypervisors/vmware/src/test/java/com/cloud/hypervisor/vmware/resource/VmwareResourceTest.java @@ -18,10 +18,15 @@ package com.cloud.hypervisor.vmware.resource; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.any; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -29,12 +34,23 @@ import static org.mockito.Mockito.when; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.EnumMap; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.cloud.hypervisor.vmware.mo.DatastoreMO; +import com.cloud.hypervisor.vmware.mo.HostDatastoreBrowserMO; +import com.cloud.hypervisor.vmware.mo.HypervisorHostHelper; +import com.cloud.hypervisor.vmware.util.VmwareHelper; +import com.vmware.vim25.FileInfo; +import com.vmware.vim25.HostDatastoreBrowserSearchResults; +import com.vmware.vim25.HostDatastoreBrowserSearchSpec; import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.junit.After; import org.junit.Before; @@ -44,6 +60,7 @@ import org.mockito.InOrder; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.mockito.Spy; @@ -189,7 +206,7 @@ public class VmwareResourceTest { @Before public void setup() throws Exception { closeable = MockitoAnnotations.openMocks(this); - storageCmd = Mockito.mock(CopyCommand.class); + storageCmd = mock(CopyCommand.class); doReturn(context).when(_resource).getServiceContext(null); when(cmd.getVirtualMachine()).thenReturn(vmSpec); @@ -417,7 +434,7 @@ public class VmwareResourceTest { @Test(expected=CloudRuntimeException.class) public void testFindVmOnDatacenterNullHyperHostReference() throws Exception { - try (MockedConstruction ignored = Mockito.mockConstruction(DatacenterMO.class)) { + try (MockedConstruction ignored = mockConstruction(DatacenterMO.class)) { _resource.findVmOnDatacenter(context, hyperHost, volume); } } @@ -425,7 +442,7 @@ public class VmwareResourceTest { @Test public void testFindVmOnDatacenter() throws Exception { when(hyperHost.getHyperHostDatacenter()).thenReturn(mor); - try (MockedConstruction ignored = Mockito.mockConstruction(DatacenterMO.class, (mock, context) -> { + try (MockedConstruction ignored = mockConstruction(DatacenterMO.class, (mock, context) -> { when(mock.findVm(VOLUME_PATH)).thenReturn(vmMo); when(mock.getMor()).thenReturn(mor); })) { @@ -466,7 +483,7 @@ public class VmwareResourceTest { GetAutoScaleMetricsCommand getAutoScaleMetricsCommand = new GetAutoScaleMetricsCommand("192.168.10.1", true, "10.10.10.10", 8080, metrics); - doReturn(vpcStats).when(vmwareResource).getVPCNetworkStats(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString()); + doReturn(vpcStats).when(vmwareResource).getVPCNetworkStats(anyString(), anyString(), anyString(), anyString()); doReturn(lbStats).when(vmwareResource).getNetworkLbStats(Mockito.nullable(String.class), Mockito.nullable(String.class), Mockito.nullable(Integer.class)); Answer answer = vmwareResource.executeRequest(getAutoScaleMetricsCommand); @@ -496,7 +513,7 @@ public class VmwareResourceTest { GetAutoScaleMetricsCommand getAutoScaleMetricsCommand = new GetAutoScaleMetricsCommand("192.168.10.1", false, "10.10.10.10", 8080, metrics); - doReturn(networkStats).when(vmwareResource).getNetworkStats(Mockito.anyString(), Mockito.anyString()); + doReturn(networkStats).when(vmwareResource).getNetworkStats(anyString(), anyString()); doReturn(lbStats).when(vmwareResource).getNetworkLbStats(Mockito.nullable(String.class), Mockito.nullable(String.class), Mockito.nullable(Integer.class)); Answer answer = vmwareResource.executeRequest(getAutoScaleMetricsCommand); @@ -561,7 +578,7 @@ public class VmwareResourceTest { @Test public void testCheckGuestOsMappingCommandFailure() throws Exception { - CheckGuestOsMappingCommand cmd = Mockito.mock(CheckGuestOsMappingCommand.class); + CheckGuestOsMappingCommand cmd = mock(CheckGuestOsMappingCommand.class); when(cmd.getGuestOsName()).thenReturn("CentOS 7.2"); when(cmd.getGuestOsHypervisorMappingName()).thenReturn("centosWrongName"); when(_resource.getHyperHost(context, null)).thenReturn(hyperHost); @@ -574,11 +591,11 @@ public class VmwareResourceTest { @Test public void testCheckGuestOsMappingCommandSuccess() throws Exception { - CheckGuestOsMappingCommand cmd = Mockito.mock(CheckGuestOsMappingCommand.class); + CheckGuestOsMappingCommand cmd = mock(CheckGuestOsMappingCommand.class); when(cmd.getGuestOsName()).thenReturn("CentOS 7.2"); when(cmd.getGuestOsHypervisorMappingName()).thenReturn("centos64Guest"); when(_resource.getHyperHost(context, null)).thenReturn(hyperHost); - GuestOsDescriptor guestOsDescriptor = Mockito.mock(GuestOsDescriptor.class); + GuestOsDescriptor guestOsDescriptor = mock(GuestOsDescriptor.class); when(hyperHost.getGuestOsDescriptor("centos64Guest")).thenReturn(guestOsDescriptor); when(guestOsDescriptor.getFullName()).thenReturn("centos64Guest"); @@ -589,7 +606,7 @@ public class VmwareResourceTest { @Test public void testCheckGuestOsMappingCommandException() { - CheckGuestOsMappingCommand cmd = Mockito.mock(CheckGuestOsMappingCommand.class); + CheckGuestOsMappingCommand cmd = mock(CheckGuestOsMappingCommand.class); when(cmd.getGuestOsName()).thenReturn("CentOS 7.2"); when(cmd.getGuestOsHypervisorMappingName()).thenReturn("centos64Guest"); when(_resource.getHyperHost(context, null)).thenReturn(null); @@ -601,7 +618,7 @@ public class VmwareResourceTest { @Test public void testGetHypervisorGuestOsNamesCommandFailure() throws Exception { - GetHypervisorGuestOsNamesCommand cmd = Mockito.mock(GetHypervisorGuestOsNamesCommand.class); + GetHypervisorGuestOsNamesCommand cmd = mock(GetHypervisorGuestOsNamesCommand.class); when(cmd.getKeyword()).thenReturn("CentOS"); when(_resource.getHyperHost(context, null)).thenReturn(hyperHost); when(hyperHost.getGuestOsDescriptors()).thenReturn(null); @@ -613,10 +630,10 @@ public class VmwareResourceTest { @Test public void testGetHypervisorGuestOsNamesCommandSuccessWithKeyword() throws Exception { - GetHypervisorGuestOsNamesCommand cmd = Mockito.mock(GetHypervisorGuestOsNamesCommand.class); + GetHypervisorGuestOsNamesCommand cmd = mock(GetHypervisorGuestOsNamesCommand.class); when(cmd.getKeyword()).thenReturn("CentOS"); when(_resource.getHyperHost(context, null)).thenReturn(hyperHost); - GuestOsDescriptor guestOsDescriptor = Mockito.mock(GuestOsDescriptor.class); + GuestOsDescriptor guestOsDescriptor = mock(GuestOsDescriptor.class); when(guestOsDescriptor.getFullName()).thenReturn("centos64Guest"); when(guestOsDescriptor.getId()).thenReturn("centos64Guest"); List guestOsDescriptors = new ArrayList<>(); @@ -631,9 +648,9 @@ public class VmwareResourceTest { @Test public void testGetHypervisorGuestOsNamesCommandSuccessWithoutKeyword() throws Exception { - GetHypervisorGuestOsNamesCommand cmd = Mockito.mock(GetHypervisorGuestOsNamesCommand.class); + GetHypervisorGuestOsNamesCommand cmd = mock(GetHypervisorGuestOsNamesCommand.class); when(_resource.getHyperHost(context, null)).thenReturn(hyperHost); - GuestOsDescriptor guestOsDescriptor = Mockito.mock(GuestOsDescriptor.class); + GuestOsDescriptor guestOsDescriptor = mock(GuestOsDescriptor.class); when(guestOsDescriptor.getFullName()).thenReturn("centos64Guest"); when(guestOsDescriptor.getId()).thenReturn("centos64Guest"); List guestOsDescriptors = new ArrayList<>(); @@ -647,8 +664,8 @@ public class VmwareResourceTest { } @Test - public void testGetHypervisorGuestOsNamesCommandException() throws Exception { - GetHypervisorGuestOsNamesCommand cmd = Mockito.mock(GetHypervisorGuestOsNamesCommand.class); + public void testGetHypervisorGuestOsNamesCommandException() { + GetHypervisorGuestOsNamesCommand cmd = mock(GetHypervisorGuestOsNamesCommand.class); when(cmd.getKeyword()).thenReturn("CentOS"); when(_resource.getHyperHost(context, null)).thenReturn(null); @@ -656,4 +673,173 @@ public class VmwareResourceTest { assertFalse(answer.getResult()); } + + @Test + public void testExecuteWithValidPath() throws Exception { + // Setup + ListDataStoreObjectsCommand cmd = new ListDataStoreObjectsCommand(destDataStoreTO, "valid/path", 0, 10); + HostDatastoreBrowserMO browserMo = mock(HostDatastoreBrowserMO.class); + HostDatastoreBrowserSearchResults results = mock(HostDatastoreBrowserSearchResults.class); + FileInfo fileInfo = mock(FileInfo.class); + List fileInfoList = new ArrayList<>(); + fileInfoList.add(fileInfo); + Date date = new Date(); + + doReturn(context).when(vmwareResource).getServiceContext(any()); + doReturn(hyperHost).when(vmwareResource).getHyperHost(context); + when(browserMo.searchDatastore(anyString(), any(HostDatastoreBrowserSearchSpec.class))).thenReturn(results); + when(results.getFile()).thenReturn(fileInfoList); + when(fileInfo.getPath()).thenReturn("file.txt"); + when(fileInfo.getFileSize()).thenReturn(1L); + when(fileInfo.getModification()).thenReturn(VmwareHelper.getXMLGregorianCalendar(date, 0)); + + // Execute + ListDataStoreObjectsAnswer answer; + try (MockedStatic ignored = mockStatic(HypervisorHostHelper.class); + MockedConstruction ignored2 = mockConstruction(DatastoreMO.class, (mock, context1) -> { + when(mock.getName()).thenReturn("datastore"); + when(mock.getHostDatastoreBrowserMO()).thenReturn(browserMo); + }) + ) { + when(HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(any(), any())).thenReturn(mor); + + answer = vmwareResource.execute(cmd); + + } + // Verify + assertNotNull(answer); + assertTrue(answer.getResult()); + assertEquals(1, answer.getCount()); + assertEquals(Collections.singletonList("file.txt"), answer.getNames()); + assertEquals(Collections.singletonList("valid/path/file.txt"), answer.getPaths()); + assertEquals(Collections.singletonList("[datastore] valid/path/file.txt"), answer.getAbsPaths()); + assertEquals(Collections.singletonList(false), answer.getIsDirs()); + assertEquals(Collections.singletonList(1L), answer.getSizes()); + assertEquals(Collections.singletonList(date.getTime()), answer.getLastModified()); + } + + @Test + public void testExecuteWithInvalidPath() throws Exception { + // Setup + ListDataStoreObjectsCommand cmd = new ListDataStoreObjectsCommand(destDataStoreTO, "invalid/path", 0, 10); + HostDatastoreBrowserMO browserMo = mock(HostDatastoreBrowserMO.class); + doReturn(context).when(vmwareResource).getServiceContext(); + doReturn(hyperHost).when(vmwareResource).getHyperHost(context); + when(browserMo.searchDatastore(anyString(), any(HostDatastoreBrowserSearchSpec.class))).thenThrow(new Exception("was not found")); + + ListDataStoreObjectsAnswer answer; + try (MockedStatic ignored = mockStatic(HypervisorHostHelper.class); + MockedConstruction ignored2 = mockConstruction(DatastoreMO.class, (mock, context1) -> { + when(mock.getName()).thenReturn("datastore"); + when(mock.fileExists(anyString())).thenReturn(false); + when(mock.getHostDatastoreBrowserMO()).thenReturn(browserMo); + }) + ) { + when(HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(any(), any())).thenReturn(mor); + + // Execute + answer = vmwareResource.execute(cmd); + } + + // Verify + assertNotNull(answer); + assertTrue(answer.getResult()); + assertEquals(0, answer.getCount()); + assertEquals(Collections.emptyList(), answer.getNames()); + assertEquals(Collections.emptyList(), answer.getPaths()); + assertEquals(Collections.emptyList(), answer.getAbsPaths()); + assertEquals(Collections.emptyList(), answer.getIsDirs()); + assertEquals(Collections.emptyList(), answer.getSizes()); + assertEquals(Collections.emptyList(), answer.getLastModified()); + } + + @Test + public void testExecuteWithRootPath() throws Exception { + // Setup + ListDataStoreObjectsCommand cmd = new ListDataStoreObjectsCommand(destDataStoreTO, "/", 0, 10); + HostDatastoreBrowserMO browserMo = mock(HostDatastoreBrowserMO.class); + HostDatastoreBrowserSearchResults results = mock(HostDatastoreBrowserSearchResults.class); + FileInfo fileInfo = mock(FileInfo.class); + List fileInfoList = new ArrayList<>(); + fileInfoList.add(fileInfo); + Date date = new Date(); + + doReturn(context).when(vmwareResource).getServiceContext(); + doReturn(hyperHost).when(vmwareResource).getHyperHost(context); + when(browserMo.searchDatastore(anyString(), any(HostDatastoreBrowserSearchSpec.class))).thenReturn(results); + when(results.getFile()).thenReturn(fileInfoList); + when(fileInfo.getPath()).thenReturn("file.txt"); + when(fileInfo.getFileSize()).thenReturn(1L); + when(fileInfo.getModification()).thenReturn(VmwareHelper.getXMLGregorianCalendar(date, 0)); + + // Execute + ListDataStoreObjectsAnswer answer; + try (MockedStatic ignored = mockStatic(HypervisorHostHelper.class); + MockedConstruction ignored2 = mockConstruction(DatastoreMO.class, (mock, context1) -> { + when(mock.getName()).thenReturn("datastore"); + when(mock.getHostDatastoreBrowserMO()).thenReturn(browserMo); + }) + ) { + when(HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(any(), any())).thenReturn(mor); + + // Execute + answer = vmwareResource.execute(cmd); + } + + // Verify + assertNotNull(answer); + assertTrue(answer.getResult()); + assertEquals(1, answer.getCount()); + assertEquals(Collections.singletonList("file.txt"), answer.getNames()); + assertEquals(Collections.singletonList("/file.txt"), answer.getPaths()); + assertEquals(Collections.singletonList("[datastore] /file.txt"), answer.getAbsPaths()); + assertEquals(Collections.singletonList(false), answer.getIsDirs()); + assertEquals(Collections.singletonList(1L), answer.getSizes()); + assertEquals(Collections.singletonList(date.getTime()), answer.getLastModified()); + } + + @Test + public void testExecuteWithEmptyPath() throws Exception { + // Setup + ListDataStoreObjectsCommand cmd = new ListDataStoreObjectsCommand(destDataStoreTO, "", 0, 10); + HostDatastoreBrowserMO browserMo = mock(HostDatastoreBrowserMO.class); + HostDatastoreBrowserSearchResults results = mock(HostDatastoreBrowserSearchResults.class); + FileInfo fileInfo = mock(FileInfo.class); + List fileInfoList = new ArrayList<>(); + fileInfoList.add(fileInfo); + Date date = new Date(); + + doReturn(context).when(vmwareResource).getServiceContext(); + doReturn(hyperHost).when(vmwareResource).getHyperHost(context); + when(browserMo.searchDatastore(anyString(), any(HostDatastoreBrowserSearchSpec.class))).thenReturn(results); + when(results.getFile()).thenReturn(fileInfoList); + when(fileInfo.getPath()).thenReturn("file.txt"); + when(fileInfo.getFileSize()).thenReturn(1L); + when(fileInfo.getModification()).thenReturn(VmwareHelper.getXMLGregorianCalendar(date, 0)); + + ListDataStoreObjectsAnswer answer; + try (MockedStatic ignored = mockStatic(HypervisorHostHelper.class); + MockedConstruction ignored2 = mockConstruction(DatastoreMO.class, (mock, context1) -> { + when(mock.getName()).thenReturn("datastore"); + when(mock.getHostDatastoreBrowserMO()).thenReturn(browserMo); + }) + ) { + when(HypervisorHostHelper.findDatastoreWithBackwardsCompatibility(any(), any())).thenReturn(mor); + + // Execute + answer = vmwareResource.execute(cmd); + + } + + // Verify + assertNotNull(answer); + assertTrue(answer.getResult()); + assertEquals(1, answer.getCount()); + assertEquals(Collections.singletonList("file.txt"), answer.getNames()); + assertEquals(Collections.singletonList("/file.txt"), answer.getPaths()); + assertEquals(Collections.singletonList("[datastore] /file.txt"), answer.getAbsPaths()); + assertEquals(Collections.singletonList(false), answer.getIsDirs()); + assertEquals(Collections.singletonList(1L), answer.getSizes()); + assertEquals(Collections.singletonList(date.getTime()), answer.getLastModified()); + } } diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java index 3227337a9ef..8db5a1159f8 100644 --- a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBase.java @@ -45,16 +45,23 @@ import java.util.Queue; import java.util.Random; import java.util.Set; import java.util.UUID; +import java.util.Vector; import java.util.concurrent.TimeoutException; import javax.naming.ConfigurationException; import javax.xml.parsers.ParserConfigurationException; +import com.trilead.ssh2.SFTPException; +import com.trilead.ssh2.SFTPv3Client; +import com.trilead.ssh2.SFTPv3DirectoryEntry; +import com.trilead.ssh2.SFTPv3FileAttributes; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageAnswer; import org.apache.cloudstack.diagnostics.CopyToSecondaryStorageCommand; import org.apache.cloudstack.diagnostics.DiagnosticsService; import org.apache.cloudstack.hypervisor.xenserver.ExtraConfigurationUtility; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.to.TemplateObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.cloudstack.utils.security.ParserUtils; @@ -5722,6 +5729,72 @@ public abstract class CitrixResourceBase extends ServerResourceBase implements S } + public Answer listFilesAtPath(ListDataStoreObjectsCommand command) throws IOException, XmlRpcException { + DataStoreTO store = command.getStore(); + int startIndex = command.getStartIndex(); + int pageSize = command.getPageSize(); + String relativePath = command.getPath(); + if (relativePath.endsWith("/")) { + relativePath = relativePath.substring(0, relativePath.length() - 1); + } + + final Connection conn = getConnection(); + + final SR sr = getStorageRepository(conn, store.getUuid()); + final com.trilead.ssh2.Connection sshConnection = new com.trilead.ssh2.Connection(_host.getIp(), 22); + try { + sshConnection.connect(null, 60000, 60000); + if (!sshConnection.authenticateWithPassword(_username, _password.peek())) { + throw new CloudRuntimeException("Unable to authenticate"); + } + String mountPoint = "/var/run/sr-mount/" + sr.getUuid(conn); + boolean pathExists = true; + SFTPv3FileAttributes fileAttr = null; + int count = 0; + List names = new ArrayList<>(); + List paths = new ArrayList<>(); + List absPaths = new ArrayList<>(); + List isDirs = new ArrayList<>(); + List sizes = new ArrayList<>(); + List modifiedList = new ArrayList<>(); + SFTPv3Client client = new SFTPv3Client(sshConnection); + try { + fileAttr = client._stat(mountPoint + "/" + relativePath); + + // Path doesn't exist + if (fileAttr == null) { + return new ListDataStoreObjectsAnswer(false, count, names, paths, absPaths, isDirs, sizes, modifiedList); + } + + try { + Vector fileList = client.ls(mountPoint + "/" + relativePath); + count = fileList.size() - 2; // -2 for . and .. + for (int i = startIndex + 2; i < startIndex + pageSize + 2 && i < fileList.size(); i++) { + SFTPv3DirectoryEntry entry = (SFTPv3DirectoryEntry) fileList.get(i); + names.add(entry.filename); + paths.add(relativePath + "/" + entry.filename); + isDirs.add(entry.attributes.isDirectory()); + sizes.add(entry.attributes.size); + modifiedList.add(entry.attributes.mtime * 1000L); + } + } catch (SFTPException e) { + // Path is a file + count = 1; + names.add(relativePath.substring(relativePath.lastIndexOf("/") + 1)); + paths.add(relativePath); + isDirs.add(false); + sizes.add(fileAttr.size); + modifiedList.add(fileAttr.mtime * 1000L); + } + } finally { + client.close(); + } + return new ListDataStoreObjectsAnswer(pathExists, count, names, paths, absPaths, isDirs, sizes, modifiedList); + } finally { + sshConnection.close(); + } + } + /** * Get Diagnostics Data API * Copy zip file from system vm and copy file directly to secondary storage diff --git a/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixListDataStoreObjectsCommandWrapper.java b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixListDataStoreObjectsCommandWrapper.java new file mode 100644 index 00000000000..1be7879d602 --- /dev/null +++ b/plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/wrapper/xenbase/CitrixListDataStoreObjectsCommandWrapper.java @@ -0,0 +1,50 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.xenserver.resource.wrapper.xenbase; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.xensource.xenapi.Types.XenAPIException; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; +import org.apache.log4j.Logger; +import org.apache.xmlrpc.XmlRpcException; + +@ResourceWrapper(handles = ListDataStoreObjectsCommand.class) +public final class CitrixListDataStoreObjectsCommandWrapper extends CommandWrapper { + + private static final Logger LOGGER = Logger.getLogger(CitrixListDataStoreObjectsCommandWrapper.class); + + @Override + public Answer execute(final ListDataStoreObjectsCommand command, final CitrixResourceBase citrixResourceBase) { + try { + return citrixResourceBase.listFilesAtPath(command); + } catch (XenAPIException e) { + LOGGER.warn("XenAPI exception", e); + + } catch (XmlRpcException e) { + LOGGER.warn("Xml Rpc Exception", e); + } catch (Exception e) { + LOGGER.warn("Caught exception", e); + } + return null; + } +} diff --git a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java index 27a108779e0..a765ddccdfd 100644 --- a/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java +++ b/plugins/hypervisors/xenserver/src/test/java/com/cloud/hypervisor/xenserver/resource/CitrixResourceBaseTest.java @@ -16,13 +16,23 @@ package com.cloud.hypervisor.xenserver.resource; import java.io.File; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Vector; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.to.DataStoreTO; +import com.trilead.ssh2.SFTPException; +import com.trilead.ssh2.SFTPv3Client; +import com.trilead.ssh2.SFTPv3DirectoryEntry; +import com.trilead.ssh2.SFTPv3FileAttributes; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.xmlrpc.XmlRpcException; import org.junit.After; import org.junit.Assert; @@ -32,6 +42,7 @@ import org.junit.runner.RunWith; import org.mockito.BDDMockito; import org.mockito.InOrder; import org.mockito.Mock; +import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.Spy; @@ -52,6 +63,7 @@ import com.xensource.xenapi.PBD; import com.xensource.xenapi.SR; import com.xensource.xenapi.Types.XenAPIException; import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; import static com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.PLATFORM_CORES_PER_SOCKET_KEY; import static org.junit.Assert.assertEquals; @@ -79,9 +91,9 @@ public class CitrixResourceBaseTest { private static final String platformString = "device-model:qemu-upstream-compat;vga:std;videoram:8;apic:true;viridian:false;timeoffset:0;pae:true;acpi:1;hpet:true;secureboot:false;nx:true"; - final static long[] vpcStats = { 1L, 2L }; - final static long[] networkStats = { 3L, 4L }; - final static long[] lbStats = { 5L }; + final static long[] vpcStats = {1L, 2L}; + final static long[] networkStats = {3L, 4L}; + final static long[] lbStats = {5L}; final static String privateIp = "192.168.1.1"; final static String publicIp = "10.10.10.10"; final static Integer port = 8080; @@ -470,4 +482,135 @@ public class CitrixResourceBaseTest { result = citrixResourceBaseSpy.networkUsage(connectionMock, null, "put", null); Assert.assertNull(result); } + + @Test + public void testListFilesAtPath() throws IOException, XmlRpcException { + SR srMock = Mockito.mock(SR.class); + SFTPv3FileAttributes fileAttributesMock = Mockito.mock(SFTPv3FileAttributes.class); + SFTPv3DirectoryEntry directoryEntryMock = Mockito.mock(SFTPv3DirectoryEntry.class); + Vector fileListMock = new Vector(); + for (int i=0; i < 16; ++i) { + fileListMock.add(directoryEntryMock); + } + + ListDataStoreObjectsCommand command = Mockito.mock(ListDataStoreObjectsCommand.class); + DataStoreTO store = Mockito.mock(DataStoreTO.class); + Mockito.when(command.getStore()).thenReturn(store); + Mockito.when(store.getUuid()).thenReturn("storeUuid"); + Mockito.when(command.getStartIndex()).thenReturn(0); + Mockito.when(command.getPageSize()).thenReturn(10); + Mockito.when(command.getPath()).thenReturn("/path/to/files"); + + Mockito.doReturn(connectionMock).when(citrixResourceBase).getConnection(); + Mockito.doReturn(srMock).when(citrixResourceBase).getStorageRepository(connectionMock, "storeUuid"); + ReflectionTestUtils.setField(directoryEntryMock, "filename", "file1"); + ReflectionTestUtils.setField(directoryEntryMock, "attributes", fileAttributesMock); + Mockito.when(fileAttributesMock.isDirectory()).thenReturn(false); + ReflectionTestUtils.setField(fileAttributesMock, "size", 1024L); + ReflectionTestUtils.setField(fileAttributesMock, "mtime", 123456789L); + + Answer answer; + try (MockedConstruction ignored = Mockito.mockConstruction(com.trilead.ssh2.Connection.class, (mock, context) -> { + Mockito.when(mock.authenticateWithPassword(Mockito.any(), Mockito.any())).thenReturn(true); + Mockito.when(mock.connect(null, 60000, 60000)).thenReturn(null); + }); MockedConstruction ignored2 = Mockito.mockConstruction(SFTPv3Client.class, (mock, context) -> { + Mockito.when(mock._stat(Mockito.anyString())).thenReturn(fileAttributesMock); + Mockito.when(mock.ls(Mockito.anyString())).thenReturn(fileListMock); + })) { + + answer = citrixResourceBase.listFilesAtPath(command); + } + + Assert.assertTrue(answer instanceof ListDataStoreObjectsAnswer); + ListDataStoreObjectsAnswer listAnswer = (ListDataStoreObjectsAnswer) answer; + Assert.assertTrue(listAnswer.isPathExists()); + Assert.assertEquals(14, listAnswer.getCount()); + Assert.assertEquals("file1", listAnswer.getNames().get(0)); + Assert.assertEquals("/path/to/files/file1", listAnswer.getPaths().get(0)); + Assert.assertFalse(listAnswer.getIsDirs().get(0)); + Assert.assertEquals(1024L, listAnswer.getSizes().get(0).longValue()); + Assert.assertEquals(123456789000L, listAnswer.getLastModified().get(0).longValue()); + } + + @Test + public void testListFilesAtPathWithNonExistentPath() throws IOException, XmlRpcException { + Connection connectionMock = Mockito.mock(Connection.class); + SR srMock = Mockito.mock(SR.class); + + ListDataStoreObjectsCommand command = Mockito.mock(ListDataStoreObjectsCommand.class); + DataStoreTO store = Mockito.mock(DataStoreTO.class); + Mockito.when(command.getStore()).thenReturn(store); + Mockito.when(store.getUuid()).thenReturn("storeUuid"); + Mockito.when(command.getStartIndex()).thenReturn(0); + Mockito.when(command.getPageSize()).thenReturn(10); + Mockito.when(command.getPath()).thenReturn("/path/to/non/existent/files"); + + Mockito.doReturn(connectionMock).when(citrixResourceBase).getConnection(); + Mockito.doReturn(srMock).when(citrixResourceBase).getStorageRepository(connectionMock, "storeUuid"); + + Answer answer; + try (MockedConstruction ignored = Mockito.mockConstruction(com.trilead.ssh2.Connection.class, (mock, context) -> { + Mockito.when(mock.authenticateWithPassword(Mockito.any(), Mockito.any())).thenReturn(true); + Mockito.when(mock.connect(null, 60000, 60000)).thenReturn(null); + }); MockedConstruction ignored2 = Mockito.mockConstruction(SFTPv3Client.class, (mock, context) -> { + Mockito.when(mock._stat(Mockito.anyString())).thenReturn(null); + })) { + + answer = citrixResourceBase.listFilesAtPath(command); + } + + Assert.assertTrue(answer instanceof ListDataStoreObjectsAnswer); + ListDataStoreObjectsAnswer listAnswer = (ListDataStoreObjectsAnswer) answer; + Assert.assertFalse(listAnswer.isPathExists()); + Assert.assertEquals(0, listAnswer.getCount()); + Assert.assertEquals(0, listAnswer.getNames().size()); + Assert.assertEquals(0, listAnswer.getPaths().size()); + Assert.assertEquals(0, listAnswer.getIsDirs().size()); + Assert.assertEquals(0, listAnswer.getSizes().size()); + Assert.assertEquals(0, listAnswer.getLastModified().size()); + } + + @Test + public void testListFilesAtPathWithFile() throws IOException, XmlRpcException { + Connection connectionMock = Mockito.mock(Connection.class); + SR srMock = Mockito.mock(SR.class); + SFTPv3FileAttributes fileAttributesMock = Mockito.mock(SFTPv3FileAttributes.class); + Vector fileListMock = new Vector(); + fileListMock.add(fileAttributesMock); + + ListDataStoreObjectsCommand command = Mockito.mock(ListDataStoreObjectsCommand.class); + DataStoreTO store = Mockito.mock(DataStoreTO.class); + Mockito.when(command.getStore()).thenReturn(store); + Mockito.when(store.getUuid()).thenReturn("storeUuid"); + Mockito.when(command.getStartIndex()).thenReturn(0); + Mockito.when(command.getPageSize()).thenReturn(10); + Mockito.when(command.getPath()).thenReturn("/path/to/file"); + + Mockito.doReturn(connectionMock).when(citrixResourceBase).getConnection(); + Mockito.doReturn(srMock).when(citrixResourceBase).getStorageRepository(connectionMock, "storeUuid"); + ReflectionTestUtils.setField(fileAttributesMock, "size", 1024L); + ReflectionTestUtils.setField(fileAttributesMock, "mtime", 123456789L); + + Answer answer; + try (MockedConstruction ignored = Mockito.mockConstruction(com.trilead.ssh2.Connection.class, (mock, context) -> { + Mockito.when(mock.authenticateWithPassword(Mockito.any(), Mockito.any())).thenReturn(true); + Mockito.when(mock.connect(null, 60000, 60000)).thenReturn(null); + }); MockedConstruction ignored2 = Mockito.mockConstruction(SFTPv3Client.class, (mock, context) -> { + Mockito.when(mock._stat(Mockito.anyString())).thenReturn(fileAttributesMock); + Mockito.when(mock.ls(Mockito.anyString())).thenThrow(Mockito.mock(SFTPException.class)); + })) { + + answer = citrixResourceBase.listFilesAtPath(command); + } + + Assert.assertTrue(answer instanceof ListDataStoreObjectsAnswer); + ListDataStoreObjectsAnswer listAnswer = (ListDataStoreObjectsAnswer) answer; + Assert.assertTrue(listAnswer.isPathExists()); + Assert.assertEquals(1, listAnswer.getCount()); + Assert.assertEquals("file", listAnswer.getNames().get(0)); + Assert.assertEquals("/path/to/file", listAnswer.getPaths().get(0)); + Assert.assertFalse(listAnswer.getIsDirs().get(0)); + Assert.assertEquals(1024L, listAnswer.getSizes().get(0).longValue()); + Assert.assertEquals(123456789000L, listAnswer.getLastModified().get(0).longValue()); + } } diff --git a/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java b/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java index 8abf802d9de..71fa2e91bcb 100644 --- a/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java +++ b/plugins/storage/image/default/src/main/java/org/apache/cloudstack/storage/datastore/driver/CloudStackImageStoreDriverImpl.java @@ -70,7 +70,12 @@ public class CloudStackImageStoreDriverImpl extends NfsImageStoreDriverImpl { EndPoint ep = _epSelector.select(store); // Create Symlink at ssvm String path = installPath; - String uuid = UUID.randomUUID().toString() + "." + format.getFileExtension(); + String uuid = UUID.randomUUID().toString(); + if (format != null) { + uuid = uuid + "." + format.getFileExtension(); + } else if (path.lastIndexOf(".") != -1) { + uuid = uuid + "." + path.substring(path.lastIndexOf(".") + 1); + } CreateEntityDownloadURLCommand cmd = new CreateEntityDownloadURLCommand(((ImageStoreEntity)store).getMountPoint(), path, uuid, dataObject == null ? null: dataObject.getTO()); Answer ans = null; @@ -82,7 +87,7 @@ public class CloudStackImageStoreDriverImpl extends NfsImageStoreDriverImpl { ans = ep.sendMessage(cmd); } if (ans == null || !ans.getResult()) { - String errorString = "Unable to create a link for entity at " + installPath + " on ssvm," + ans.getDetails(); + String errorString = "Unable to create a link for entity at " + installPath + " on ssvm, " + ans.getDetails(); s_logger.error(errorString); throw new CloudRuntimeException(errorString); } diff --git a/server/src/main/java/com/cloud/api/ApiDBUtils.java b/server/src/main/java/com/cloud/api/ApiDBUtils.java index 5d821d38f2e..026c8f03407 100644 --- a/server/src/main/java/com/cloud/api/ApiDBUtils.java +++ b/server/src/main/java/com/cloud/api/ApiDBUtils.java @@ -2121,12 +2121,12 @@ public class ApiDBUtils { return s_templateJoinDao.newTemplateResponse(detailsView, view, vr); } - public static SnapshotResponse newSnapshotResponse(ResponseView view, boolean isShowUnique, SnapshotJoinVO vr) { - return s_snapshotJoinDao.newSnapshotResponse(view, isShowUnique, vr); + public static TemplateResponse newIsoResponse(TemplateJoinVO vr, ResponseView view) { + return s_templateJoinDao.newIsoResponse(vr, view); } - public static TemplateResponse newIsoResponse(TemplateJoinVO vr) { - return s_templateJoinDao.newIsoResponse(vr); + public static SnapshotResponse newSnapshotResponse(ResponseView view, boolean isShowUnique, SnapshotJoinVO vr) { + return s_snapshotJoinDao.newSnapshotResponse(view, isShowUnique, vr); } public static TemplateResponse fillTemplateDetails(EnumSet detailsView, ResponseView view, TemplateResponse vrData, TemplateJoinVO vr) { diff --git a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java index 93b858077f6..c99aa293234 100644 --- a/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java +++ b/server/src/main/java/com/cloud/api/query/QueryManagerImpl.java @@ -34,6 +34,8 @@ import java.util.stream.Stream; import javax.inject.Inject; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.host.Host; import com.cloud.host.dao.HostDao; import com.cloud.network.as.AutoScaleVmGroupVmMapVO; @@ -76,6 +78,7 @@ import org.apache.cloudstack.api.command.admin.management.ListMgmtsCmd; import org.apache.cloudstack.api.command.admin.resource.icon.ListResourceIconCmd; import org.apache.cloudstack.api.command.admin.router.GetRouterHealthCheckResultsCmd; import org.apache.cloudstack.api.command.admin.router.ListRoutersCmd; +import org.apache.cloudstack.api.command.admin.snapshot.ListSnapshotsCmdByAdmin; import org.apache.cloudstack.api.command.admin.storage.ListImageStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListSecondaryStagingStoresCmd; import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; @@ -149,9 +152,12 @@ import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import org.apache.cloudstack.query.QueryService; import org.apache.cloudstack.resourcedetail.dao.DiskOfferingDetailsDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolDetailsDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.EnumUtils; import org.apache.commons.lang3.StringUtils; @@ -475,6 +481,15 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q @Inject private ProjectInvitationDao projectInvitationDao; + @Inject + private TemplateDataStoreDao templateDataStoreDao; + + @Inject + private VMTemplatePoolDao templatePoolDao; + + @Inject + private SnapshotDataStoreDao snapshotDataStoreDao; + @Inject private UserDao userDao; @@ -3799,8 +3814,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } List permittedAccountIds = new ArrayList(); - Ternary domainIdRecursiveListProject = new Ternary(cmd.getDomainId(), cmd.isRecursive(), null); - accountMgr.buildACLSearchParameters(caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccountIds, domainIdRecursiveListProject, listAll, false); + Ternary domainIdRecursiveListProject = new Ternary<>(cmd.getDomainId(), cmd.isRecursive(), null); + accountMgr.buildACLSearchParameters( + caller, id, cmd.getAccountName(), cmd.getProjectId(), permittedAccountIds, + domainIdRecursiveListProject, listAll, false + ); ListProjectResourcesCriteria listProjectResourcesCriteria = domainIdRecursiveListProject.third(); List permittedAccounts = new ArrayList(); for (Long accountId : permittedAccountIds) { @@ -3820,13 +3838,19 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } Boolean isVnf = cmd.getVnf(); - return searchForTemplatesInternal(id, cmd.getTemplateName(), cmd.getKeyword(), templateFilter, false, null, cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), hypervisorType, - showDomr, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedTmpl, cmd.getIds(), parentTemplateId, cmd.getShowUnique(), templateType, isVnf); + return searchForTemplatesInternal(id, cmd.getTemplateName(), cmd.getKeyword(), templateFilter, false, + null, cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), cmd.getStoragePoolId(), + cmd.getImageStoreId(), hypervisorType, showDomr, cmd.listInReadyState(), permittedAccounts, caller, + listProjectResourcesCriteria, tags, showRemovedTmpl, cmd.getIds(), parentTemplateId, cmd.getShowUnique(), + templateType, isVnf); } - private Pair, Integer> searchForTemplatesInternal(Long templateId, String name, String keyword, TemplateFilter templateFilter, boolean isIso, Boolean bootable, Long pageSize, - Long startIndex, Long zoneId, HypervisorType hyperType, boolean showDomr, boolean onlyReady, List permittedAccounts, Account caller, - ListProjectResourcesCriteria listProjectResourcesCriteria, Map tags, boolean showRemovedTmpl, List ids, Long parentTemplateId, Boolean showUnique, String templateType, + private Pair, Integer> searchForTemplatesInternal(Long templateId, String name, String keyword, + TemplateFilter templateFilter, boolean isIso, Boolean bootable, Long pageSize, + Long startIndex, Long zoneId, Long storagePoolId, Long imageStoreId, HypervisorType hyperType, + boolean showDomr, boolean onlyReady, List permittedAccounts, Account caller, + ListProjectResourcesCriteria listProjectResourcesCriteria, Map tags, + boolean showRemovedTmpl, List ids, Long parentTemplateId, Boolean showUnique, String templateType, Boolean isVnf) { // check if zone is configured, if not, just return empty list @@ -3852,8 +3876,23 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q if (ids != null && !ids.isEmpty()) { sb.and("idIN", sb.entity().getId(), SearchCriteria.Op.IN); } + + if (storagePoolId != null) { + SearchBuilder storagePoolSb = templatePoolDao.createSearchBuilder(); + storagePoolSb.and("pool_id", storagePoolSb.entity().getPoolId(), SearchCriteria.Op.EQ); + sb.join("storagePool", storagePoolSb, storagePoolSb.entity().getTemplateId(), sb.entity().getId(), JoinBuilder.JoinType.INNER); + } + SearchCriteria sc = sb.create(); + if (imageStoreId != null) { + sc.addAnd("dataStoreId", SearchCriteria.Op.EQ, imageStoreId); + } + + if (storagePoolId != null) { + sc.setJoinParameters("storagePool", "pool_id", storagePoolId); + } + // verify templateId parameter and specially handle it if (templateId != null) { template = _templateDao.findByIdIncludingRemoved(templateId); // Done for backward compatibility - Bug-5221 @@ -3995,7 +4034,6 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q return templateChecks(isIso, hypers, tags, name, keyword, hyperType, onlyReady, bootable, zoneId, showDomr, caller, showRemovedTmpl, parentTemplateId, showUnique, templateType, isVnf, searchFilter, sc); - } /** @@ -4154,7 +4192,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q } else { sc.addAnd("templateState", SearchCriteria.Op.IN, new State[] {State.Active, State.UploadAbandoned, State.UploadError, State.NotUploaded, State.UploadInProgress}); if (showUnique) { - final String[] distinctColumns = {"id"}; + final String[] distinctColumns = {"template_view.id"}; uniqueTmplPair = _templateJoinDao.searchAndDistinctCount(sc, searchFilter, distinctColumns); } else { final String[] distinctColumns = {"temp_zone_pair"}; @@ -4233,8 +4271,10 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q HypervisorType hypervisorType = HypervisorType.getType(cmd.getHypervisor()); - return searchForTemplatesInternal(cmd.getId(), cmd.getIsoName(), cmd.getKeyword(), isoFilter, true, cmd.isBootable(), cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), - hypervisorType, true, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, tags, showRemovedISO, null, null, cmd.getShowUnique(), null, null); + return searchForTemplatesInternal(cmd.getId(), cmd.getIsoName(), cmd.getKeyword(), isoFilter, true, cmd.isBootable(), + cmd.getPageSizeVal(), cmd.getStartIndex(), cmd.getZoneId(), cmd.getStoragePoolId(), cmd.getImageStoreId(), + hypervisorType, true, cmd.listInReadyState(), permittedAccounts, caller, listProjectResourcesCriteria, + tags, showRemovedISO, null, null, cmd.getShowUnique(), null, null); } @Override @@ -4700,11 +4740,11 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Pair, Integer> result = searchForSnapshotsWithParams(cmd.getId(), cmd.getIds(), cmd.getVolumeId(), cmd.getSnapshotName(), cmd.getKeyword(), cmd.getTags(), cmd.getSnapshotType(), cmd.getIntervalType(), cmd.getZoneId(), cmd.getLocationType(), - cmd.isShowUnique(), cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId(), - cmd.getStartIndex(), cmd.getPageSizeVal(), cmd.listAll(), cmd.isRecursive(), caller); + cmd.isShowUnique(), cmd.getAccountName(), cmd.getDomainId(), cmd.getProjectId(), cmd.getStoragePoolId(), + cmd.getImageStoreId(), cmd.getStartIndex(), cmd.getPageSizeVal(), cmd.listAll(), cmd.isRecursive(), caller); ListResponse response = new ListResponse<>(); ResponseView respView = ResponseView.Restricted; - if (CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) { + if (cmd instanceof ListSnapshotsCmdByAdmin) { respView = ResponseView.Full; } List templateResponses = ViewResponseHelper.createSnapshotResponse(respView, cmd.isShowUnique(), result.first().toArray(new SnapshotJoinVO[result.first().size()])); @@ -4719,7 +4759,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q Pair, Integer> result = searchForSnapshotsWithParams(cmd.getId(), null, null, null, null, null, null, null, zoneIds.get(0), Snapshot.LocationType.SECONDARY.name(), - false, null, null, null, + false, null, null, null, null, null, null, null, true, false, caller); ResponseView respView = ResponseView.Restricted; if (CallContext.current().getCallingAccount().getType() == Account.Type.ADMIN) { @@ -4734,7 +4774,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q private Pair, Integer> searchForSnapshotsWithParams(final Long id, List ids, final Long volumeId, final String name, final String keyword, final Map tags, final String snapshotTypeStr, final String intervalTypeStr, final Long zoneId, final String locationTypeStr, - final boolean isShowUnique, final String accountName, Long domainId, final Long projectId, + final boolean isShowUnique, final String accountName, Long domainId, final Long projectId, final Long storagePoolId, final Long imageStoreId, final Long startIndex, final Long pageSize,final boolean listAll, boolean isRecursive, final Account caller) { ids = getIdsListFromCmd(id, ids); Snapshot.LocationType locationType = null; @@ -4778,6 +4818,7 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sb.and("snapshotTypeNEQ", sb.entity().getSnapshotType(), SearchCriteria.Op.NIN); sb.and("dataCenterId", sb.entity().getDataCenterId(), SearchCriteria.Op.EQ); sb.and("locationType", sb.entity().getStoreRole(), SearchCriteria.Op.EQ); + sb.and("imageStoreId", sb.entity().getStoreId(), SearchCriteria.Op.EQ); if (tags != null && !tags.isEmpty()) { SearchBuilder tagSearch = resourceTagDao.createSearchBuilder(); @@ -4791,11 +4832,28 @@ public class QueryManagerImpl extends MutualExclusiveIdsManagerBase implements Q sb.join("tagSearch", tagSearch, sb.entity().getId(), tagSearch.entity().getResourceId(), JoinBuilder.JoinType.INNER); } + if (storagePoolId != null) { + SearchBuilder storagePoolSb = snapshotDataStoreDao.createSearchBuilder(); + storagePoolSb.and("poolId", storagePoolSb.entity().getDataStoreId(), SearchCriteria.Op.EQ); + storagePoolSb.and("role", storagePoolSb.entity().getRole(), SearchCriteria.Op.EQ); + sb.join("storagePoolSb", storagePoolSb, sb.entity().getId(), storagePoolSb.entity().getSnapshotId(), JoinBuilder.JoinType.INNER); + } + SearchCriteria sc = sb.create(); accountMgr.buildACLSearchCriteria(sc, domainId, isRecursive, permittedAccountIds, listProjectResourcesCriteria); sc.setParameters("statusNEQ", Snapshot.State.Destroyed); + if (imageStoreId != null) { + sc.setParameters("imageStoreId", imageStoreId); + locationType = Snapshot.LocationType.SECONDARY; + } + + if (storagePoolId != null) { + sc.setJoinParameters("storagePoolSb", "poolId", storagePoolId); + sc.setJoinParameters("storagePoolSb", "role", DataStoreRole.Image); + } + if (volumeId != null) { sc.setParameters("volumeId", volumeId); } diff --git a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java index b415699ab19..b909d69e80e 100644 --- a/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java +++ b/server/src/main/java/com/cloud/api/query/ViewResponseHelper.java @@ -633,7 +633,7 @@ public class ViewResponseHelper { TemplateResponse vrData = vrDataList.get(vr.getTempZonePair()); if (vrData == null) { // first time encountering this volume - vrData = ApiDBUtils.newIsoResponse(vr); + vrData = ApiDBUtils.newIsoResponse(vr, view); } else { // update tags vrData = ApiDBUtils.fillTemplateDetails(EnumSet.of(DomainDetails.all), view, vrData, vr); diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java index 1b7edd32592..a7b82e47265 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDao.java @@ -34,7 +34,7 @@ public interface TemplateJoinDao extends GenericDao { TemplateResponse newTemplateResponse(EnumSet detailsView, ResponseView view, TemplateJoinVO tmpl); - TemplateResponse newIsoResponse(TemplateJoinVO tmpl); + TemplateResponse newIsoResponse(TemplateJoinVO tmpl, ResponseView view); TemplateResponse newUpdateResponse(TemplateJoinVO tmpl); diff --git a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java index b9dcad98f22..501d413f117 100644 --- a/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java +++ b/server/src/main/java/com/cloud/api/query/dao/TemplateJoinDaoImpl.java @@ -24,12 +24,16 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import javax.inject.Inject; import com.cloud.deployasis.DeployAsIsConstants; import com.cloud.deployasis.TemplateDeployAsIsDetailVO; import com.cloud.deployasis.dao.TemplateDeployAsIsDetailsDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.dao.VMTemplatePoolDao; import com.cloud.storage.VnfTemplateDetailVO; import com.cloud.storage.VnfTemplateNicVO; import com.cloud.storage.dao.VnfTemplateDetailsDao; @@ -41,6 +45,8 @@ import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.response.VnfNicResponse; import org.apache.cloudstack.api.response.VnfTemplateResponse; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -93,6 +99,10 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation downloadDetailInImageStores = null; for (TemplateDataStoreVO templateInStore : templatesInStore) { downloadDetailInImageStores = new HashMap<>(); - ImageStoreVO datastore = dataStoreDao.findById(templateInStore.getDataStoreId()); - if (datastore != null) { - downloadDetailInImageStores.put("datastore", datastore.getName()); + ImageStoreVO imageStore = dataStoreDao.findById(templateInStore.getDataStoreId()); + if (imageStore != null) { + downloadDetailInImageStores.put("datastore", imageStore.getName()); + if (view.equals(ResponseView.Full)) { + downloadDetailInImageStores.put("datastoreId", imageStore.getUuid()); + downloadDetailInImageStores.put("datastoreRole", imageStore.getRole().name()); + } downloadDetailInImageStores.put("downloadPercent", Integer.toString(templateInStore.getDownloadPercent())); downloadDetailInImageStores.put("downloadState", (templateInStore.getDownloadState() != null ? templateInStore.getDownloadState().toString() : "")); downloadProgressDetails.add(downloadDetailInImageStores); } } + List poolsInZone = primaryDataStoreDao.listByDataCenterId(template.getDataCenterId()); + List poolIds = poolsInZone.stream().map(StoragePoolVO::getId).collect(Collectors.toList()); + List templatesInPool = templatePoolDao.listByTemplateId(template.getId(), poolIds); + + for (VMTemplateStoragePoolVO templateInPool : templatesInPool) { + downloadDetailInImageStores = new HashMap<>(); + StoragePoolVO storagePool = primaryDataStoreDao.findById(templateInPool.getDataStoreId()); + if (storagePool != null) { + downloadDetailInImageStores.put("datastore", storagePool.getName()); + if (view.equals(ResponseView.Full)) { + downloadDetailInImageStores.put("datastoreId", storagePool.getUuid()); + downloadDetailInImageStores.put("datastoreRole", DataStoreRole.Primary.name()); + } + downloadDetailInImageStores.put("downloadPercent", Integer.toString(templateInPool.getDownloadPercent())); + downloadDetailInImageStores.put("downloadState", (templateInPool.getDownloadState() != null ? templateInPool.getDownloadState().toString() : "")); + downloadProgressDetails.add(downloadDetailInImageStores); + } + } + TemplateResponse templateResponse = initTemplateResponse(template); templateResponse.setDownloadProgress(downloadProgressDetails); templateResponse.setId(template.getUuid()); @@ -428,7 +461,7 @@ public class TemplateJoinDaoImpl extends GenericDaoBaseWithTagInformation isosInStore = _templateStoreDao.listByTemplateNotBypassed(iso.getId()); + List> downloadProgressDetails = new ArrayList<>(); + HashMap downloadDetailInImageStores = null; + for (TemplateDataStoreVO isoInStore : isosInStore) { + downloadDetailInImageStores = new HashMap<>(); + ImageStoreVO imageStore = dataStoreDao.findById(isoInStore.getDataStoreId()); + if (imageStore != null) { + downloadDetailInImageStores.put("datastore", imageStore.getName()); + if (view.equals(ResponseView.Full)) { + downloadDetailInImageStores.put("datastoreId", imageStore.getUuid()); + downloadDetailInImageStores.put("datastoreRole", imageStore.getRole().name()); + } + downloadDetailInImageStores.put("downloadPercent", Integer.toString(isoInStore.getDownloadPercent())); + downloadDetailInImageStores.put("downloadState", (isoInStore.getDownloadState() != null ? isoInStore.getDownloadState().toString() : "")); + downloadProgressDetails.add(downloadDetailInImageStores); + } + } + + List poolsInZone = primaryDataStoreDao.listByDataCenterId(iso.getDataCenterId()); + List poolIds = poolsInZone.stream().map(StoragePoolVO::getId).collect(Collectors.toList()); + List isosInPool = templatePoolDao.listByTemplateId(iso.getId(), poolIds); + + for (VMTemplateStoragePoolVO isoInPool : isosInPool) { + downloadDetailInImageStores = new HashMap<>(); + StoragePoolVO storagePool = primaryDataStoreDao.findById(isoInPool.getDataStoreId()); + if (storagePool != null) { + downloadDetailInImageStores.put("datastore", storagePool.getName()); + if (view.equals(ResponseView.Full)) { + downloadDetailInImageStores.put("datastoreId", storagePool.getUuid()); + downloadDetailInImageStores.put("datastoreRole", DataStoreRole.Primary.name()); + } + downloadDetailInImageStores.put("downloadPercent", Integer.toString(isoInPool.getDownloadPercent())); + downloadDetailInImageStores.put("downloadState", (isoInPool.getDownloadState() != null ? isoInPool.getDownloadState().toString() : "")); + downloadProgressDetails.add(downloadDetailInImageStores); + } + } + isoResponse.setDownloadProgress(downloadProgressDetails); } if (iso.getDataCenterId() > 0) { diff --git a/server/src/main/java/com/cloud/server/ManagementServerImpl.java b/server/src/main/java/com/cloud/server/ManagementServerImpl.java index 50824d11d58..5be85d051f4 100644 --- a/server/src/main/java/com/cloud/server/ManagementServerImpl.java +++ b/server/src/main/java/com/cloud/server/ManagementServerImpl.java @@ -205,6 +205,7 @@ import org.apache.cloudstack.api.command.admin.router.StartRouterCmd; import org.apache.cloudstack.api.command.admin.router.StopRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterCmd; import org.apache.cloudstack.api.command.admin.router.UpgradeRouterTemplateCmd; +import org.apache.cloudstack.api.command.admin.snapshot.ListSnapshotsCmdByAdmin; import org.apache.cloudstack.api.command.admin.storage.AddImageStoreCmd; import org.apache.cloudstack.api.command.admin.storage.AddImageStoreS3CMD; import org.apache.cloudstack.api.command.admin.storage.CancelPrimaryStorageMaintenanceCmd; @@ -220,6 +221,7 @@ import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolsCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageProvidersCmd; import org.apache.cloudstack.api.command.admin.storage.ListStorageTagsCmd; import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; +import org.apache.cloudstack.api.command.admin.storage.MigrateResourcesToAnotherSecondaryStorageCmd; import org.apache.cloudstack.api.command.admin.storage.PreparePrimaryStorageForMaintenanceCmd; import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd; import org.apache.cloudstack.api.command.admin.storage.UpdateCloudToUseObjectStoreCmd; @@ -3804,6 +3806,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(CopyTemplateCmdByAdmin.class); cmdList.add(RegisterTemplateCmdByAdmin.class); cmdList.add(ListTemplatePermissionsCmdByAdmin.class); + cmdList.add(ListSnapshotsCmdByAdmin.class); cmdList.add(RegisterIsoCmdByAdmin.class); cmdList.add(CopyIsoCmdByAdmin.class); cmdList.add(ListIsosCmdByAdmin.class); @@ -3867,6 +3870,7 @@ public class ManagementServerImpl extends ManagerBase implements ManagementServe cmdList.add(GetRouterHealthCheckResultsCmd.class); cmdList.add(StartRollingMaintenanceCmd.class); cmdList.add(MigrateSecondaryStorageDataCmd.class); + cmdList.add(MigrateResourcesToAnotherSecondaryStorageCmd.class); cmdList.add(UploadResourceIconCmd.class); cmdList.add(DeleteResourceIconCmd.class); cmdList.add(ListResourceIconCmd.class); diff --git a/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java b/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java index 43f9cd455be..a92b75e1e1c 100644 --- a/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java +++ b/server/src/main/java/com/cloud/storage/ImageStoreServiceImpl.java @@ -20,10 +20,13 @@ package com.cloud.storage; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.naming.ConfigurationException; +import com.cloud.utils.db.UUIDManager; +import org.apache.cloudstack.api.command.admin.storage.MigrateResourcesToAnotherSecondaryStorageCmd; import org.apache.cloudstack.api.command.admin.storage.MigrateSecondaryStorageDataCmd; import org.apache.cloudstack.api.response.MigrationResponse; import org.apache.cloudstack.context.CallContext; @@ -53,6 +56,9 @@ public class ImageStoreServiceImpl extends ManagerBase implements ImageStoreServ @Inject private StorageOrchestrationService stgService; + @Inject + public UUIDManager uuidMgr; + ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, "image.store.imbalance.threshold", "0.3", @@ -147,10 +153,67 @@ public class ImageStoreServiceImpl extends ManagerBase implements ImageStoreServ return new MigrationResponse(message, policy.toString(), false); } - CallContext.current().setEventDetails("Migrating files/data objects " + "from : " + imagestores.get(0) + " to: " + imagestores.subList(1, imagestores.size())); + CallContext.current().setEventDetails("Migrating files/data objects from : " + imagestores.get(0) + " to: " + imagestores.subList(1, imagestores.size())); return stgService.migrateData(srcImgStoreId, destDatastores, policy); } + @Override + @ActionEvent(eventType = EventTypes.EVENT_IMAGE_STORE_RESOURCES_MIGRATE, eventDescription = "migrating Image store resources to another image store", async = true) + public MigrationResponse migrateResources(MigrateResourcesToAnotherSecondaryStorageCmd cmd) { + if (isMigrateJobRunning()){ + return new MigrationResponse("A migrate job is in progress, please try again later.", null, false); + } + + Long srcImgStoreId = cmd.getId(); + Long destImgStoreId = cmd.getDestStoreId(); + + if (srcImgStoreId.equals(destImgStoreId)) { + throw new InvalidParameterValueException("Source and destination image stores cannot be same"); + } + + ImageStoreVO srcImageStore = imageStoreDao.findById(srcImgStoreId); + ImageStoreVO destImageStore = imageStoreDao.findById(destImgStoreId); + + if (srcImageStore == null) { + throw new CloudRuntimeException("Cannot find secondary storage with id: " + srcImgStoreId); + } + if (destImageStore == null) { + throw new CloudRuntimeException("Cannot find secondary storage with id: " + srcImgStoreId); + } + + if (srcImageStore.getRole() != DataStoreRole.Image) { + throw new CloudRuntimeException("Source Secondary storage is not of Image Role"); + } + + if (destImageStore.getRole() != DataStoreRole.Image) { + throw new CloudRuntimeException("Destination Secondary storage is not of Image Role"); + } + + if (!srcImageStore.getProviderName().equals(DataStoreProvider.NFS_IMAGE) || !destImageStore.getProviderName().equals(DataStoreProvider.NFS_IMAGE)) { + throw new InvalidParameterValueException("Migration of datastore objects is supported only for NFS based image stores"); + } + + if (destImageStore.isReadonly()) { + throw new InvalidParameterValueException("Destination image store is read-only. Cannot migrate resources to it"); + } + + if (srcImageStore.getDataCenterId() != null && destImageStore.getDataCenterId() != null && !srcImageStore.getDataCenterId().equals(destImageStore.getDataCenterId())) { + throw new InvalidParameterValueException("Source and destination stores are not in the same zone."); + } + + List templateIdList = cmd.getTemplateIdList(); + List snapshotIdList = cmd.getSnapshotIdList(); + List templateUuidList = templateIdList.stream().map((id) -> uuidMgr.getUuid(VMTemplateVO.class, id)).collect(Collectors.toList()); + List snapshotUuidList = snapshotIdList.stream().map((id) -> uuidMgr.getUuid(SnapshotVO.class, id)).collect(Collectors.toList()); + CallContext.current().setEventDetails( + "Migrating templates (" + String.join(", ", templateUuidList) + + ") and snapshots (" + String.join(", ", snapshotUuidList) + + ") from : " + srcImageStore.getName() + " to: " + destImageStore.getName() + ); + + return stgService.migrateResources(srcImgStoreId, destImgStoreId, templateIdList, snapshotIdList); + } + // Ensures that only one migrate job may occur at a time, in order to reduce load private boolean isMigrateJobRunning() { diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index 0618a0f5104..82a6c159565 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -101,6 +101,8 @@ import org.apache.cloudstack.storage.command.SyncVolumePathAnswer; import org.apache.cloudstack.storage.command.SyncVolumePathCommand; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDetailsDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; @@ -115,6 +117,7 @@ import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.time.DateUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Logger; import org.springframework.stereotype.Component; @@ -273,6 +276,8 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C @Inject protected ImageStoreDetailsDao _imageStoreDetailsDao = null; @Inject + protected ImageStoreObjectDownloadDao _imageStoreObjectDownloadDao = null; + @Inject protected SnapshotDataStoreDao _snapshotStoreDao = null; @Inject protected TemplateDataStoreDao _templateStoreDao = null; @@ -3371,6 +3376,18 @@ public class StorageManagerImpl extends ManagerBase implements StorageManager, C s_logger.warn("caught exception while deleting download url " + templateOnImageStore.getExtractUrl() + " for template id " + templateOnImageStore.getTemplateId(), th); } } + + Date date = DateUtils.addSeconds(new Date(), -1 * _downloadUrlExpirationInterval); + List imageStoreObjectDownloadList = _imageStoreObjectDownloadDao.listToExpire(date); + for (ImageStoreObjectDownloadVO imageStoreObjectDownloadVO : imageStoreObjectDownloadList) { + try { + ImageStoreEntity secStore = (ImageStoreEntity)_dataStoreMgr.getDataStore(imageStoreObjectDownloadVO.getStoreId(), DataStoreRole.Image); + secStore.deleteExtractUrl(imageStoreObjectDownloadVO.getPath(), imageStoreObjectDownloadVO.getDownloadUrl(), null); + _imageStoreObjectDownloadDao.expunge(imageStoreObjectDownloadVO.getId()); + } catch (Throwable th) { + s_logger.warn("caught exception while deleting download url " + imageStoreObjectDownloadVO.getDownloadUrl() + " for object id " + imageStoreObjectDownloadVO.getId(), th); + } + } } // get bytesReadRate from service_offering, disk_offering and vm.disk.throttling.bytes_read_rate diff --git a/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java b/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java index 115ee718fbe..f13f6e051ad 100644 --- a/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java +++ b/server/src/main/java/org/apache/cloudstack/diagnostics/to/DiagnosticsDataTO.java @@ -33,6 +33,10 @@ public class DiagnosticsDataTO implements DataTO { this.dataStoreTO = dataStoreTO; } + public DiagnosticsDataTO(DataStoreTO dataStoreTO) { + this.dataStoreTO = dataStoreTO; + } + @Override public DataObjectType getObjectType() { return DataObjectType.ARCHIVE; diff --git a/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java b/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java new file mode 100644 index 00000000000..8828ac486f5 --- /dev/null +++ b/server/src/main/java/org/apache/cloudstack/storage/browser/StorageBrowserImpl.java @@ -0,0 +1,413 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.browser; + +import com.cloud.agent.api.Answer; +import com.cloud.api.query.MutualExclusiveIdsManagerBase; +import com.cloud.api.query.dao.ImageStoreJoinDao; +import com.cloud.api.query.vo.ImageStoreJoinVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Snapshot; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.Upload; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.admin.storage.DownloadImageStoreObjectCmd; +import org.apache.cloudstack.api.command.admin.storage.ListImageStoreObjectsCmd; +import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolObjectsCmd; +import org.apache.cloudstack.api.response.ExtractResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.context.CallContext; +import org.apache.cloudstack.diagnostics.to.DiagnosticsDataObject; +import org.apache.cloudstack.diagnostics.to.DiagnosticsDataTO; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; +import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadDao; +import org.apache.cloudstack.storage.datastore.db.ImageStoreObjectDownloadVO; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreVO; +import org.apache.cloudstack.storage.image.datastore.ImageStoreEntity; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.EnumUtils; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Component +public class StorageBrowserImpl extends MutualExclusiveIdsManagerBase implements StorageBrowser { + + @Inject + ImageStoreJoinDao imageStoreJoinDao; + + @Inject + ImageStoreObjectDownloadDao imageStoreObjectDownloadDao; + + @Inject + DataStoreManager dataStoreMgr; + + @Inject + TemplateDataStoreDao templateDataStoreDao; + + @Inject + SnapshotDataStoreDao snapshotDataStoreDao; + + @Inject + SnapshotDao snapshotDao; + + @Inject + EndPointSelector endPointSelector; + + @Inject + VMTemplatePoolDao templatePoolDao; + + @Inject + VMTemplateDao templateDao; + + @Inject + VolumeDao volumeDao; + + @Inject + VolumeDataStoreDao volumeDataStoreDao; + + @Override + public List> getCommands() { + List> cmdList = new ArrayList<>(); + cmdList.add(ListImageStoreObjectsCmd.class); + cmdList.add(ListStoragePoolObjectsCmd.class); + cmdList.add(DownloadImageStoreObjectCmd.class); + return cmdList; + } + + @Override + public ListResponse listImageStoreObjects(ListImageStoreObjectsCmd cmd) { + Long imageStoreId = cmd.getStoreId(); + String path = cmd.getPath(); + + ImageStoreJoinVO imageStore = imageStoreJoinDao.findById(imageStoreId); + DataStore dataStore = dataStoreMgr.getDataStore(imageStoreId, imageStore.getRole()); + ListDataStoreObjectsAnswer answer = listObjectsInStore(dataStore, path, cmd.getStartIndex().intValue(), cmd.getPageSizeVal().intValue()); + + return getResponse(dataStore, answer); + } + + @Override + public ListResponse listPrimaryStoreObjects(ListStoragePoolObjectsCmd cmd) { + Long storeId = cmd.getStoreId(); + String path = cmd.getPath(); + + DataStore dataStore = dataStoreMgr.getDataStore(storeId, DataStoreRole.Primary); + ListDataStoreObjectsAnswer answer = listObjectsInStore(dataStore, path, cmd.getStartIndex().intValue(), cmd.getPageSizeVal().intValue()); + + return getResponse(dataStore, answer); + } + + @Override + public ExtractResponse downloadImageStoreObject(DownloadImageStoreObjectCmd cmd) { + ImageStoreEntity imageStore = (ImageStoreEntity) dataStoreMgr.getDataStore(cmd.getStoreId(), DataStoreRole.Image); + + String path = cmd.getPath(); + if (path.startsWith("/")) { + path = path.substring(1); + } + + ImageStoreObjectDownloadVO imageStoreObj = imageStoreObjectDownloadDao.findByStoreIdAndPath(cmd.getStoreId(), path); + + if (imageStoreObj == null) { + try { + String fileExt = path.substring(path.lastIndexOf(".") + 1); + Storage.ImageFormat format = EnumUtils.getEnumIgnoreCase(Storage.ImageFormat.class, fileExt); + + DiagnosticsDataTO dataTO = new DiagnosticsDataTO(imageStore.getTO()); + DiagnosticsDataObject dataObject = new DiagnosticsDataObject(dataTO, imageStore); + String downloadUrl = imageStore.createEntityExtractUrl(path, format, dataObject); + imageStoreObj = imageStoreObjectDownloadDao.persist(new ImageStoreObjectDownloadVO(imageStore.getId(), path, downloadUrl)); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to create download url for image store object", e); + } + } + ExtractResponse response = new ExtractResponse(null, null, CallContext.current().getCallingAccountUuid(), null, null); + if (imageStoreObj != null) { + response.setUrl(imageStoreObj.getDownloadUrl()); + response.setName(cmd.getPath().substring(cmd.getPath().lastIndexOf("/") + 1)); + response.setState(Upload.Status.DOWNLOAD_URL_CREATED.toString()); + } else { + response.setState(Upload.Status.DOWNLOAD_URL_NOT_CREATED.toString()); + } + return response; + } + + ListDataStoreObjectsAnswer listObjectsInStore(DataStore dataStore, String path, int startIndex, int pageSize) { + EndPoint ep = endPointSelector.select(dataStore); + + if (ep == null) { + throw new CloudRuntimeException("No remote endpoint to send command"); + } + + ListDataStoreObjectsCommand listDSCmd = new ListDataStoreObjectsCommand(dataStore.getTO(), path, startIndex, pageSize); + listDSCmd.setWait(15); + Answer answer = null; + try { + answer = ep.sendMessage(listDSCmd); + } catch (Exception e) { + throw new CloudRuntimeException("Failed to list datastore objects", e); + } + + if (answer == null || !answer.getResult() || !(answer instanceof ListDataStoreObjectsAnswer)) { + throw new CloudRuntimeException("Failed to list datastore objects"); + } + + ListDataStoreObjectsAnswer dsAnswer = (ListDataStoreObjectsAnswer) answer; + if (!dsAnswer.isPathExists()) { + throw new IllegalArgumentException("Path " + path + " doesn't exist in store: " + dataStore.getUuid()); + } + return dsAnswer; + } + + ListResponse getResponse(DataStore dataStore, ListDataStoreObjectsAnswer answer) { + List responses = new ArrayList<>(); + + List paths = getFormattedPaths(answer.getPaths()); + List absPaths = answer.getAbsPaths(); + + Map pathSnapshotMap; + + Map pathTemplateMap; + + Map pathVolumeMap; + + if (dataStore.getRole() != DataStoreRole.Primary) { + pathTemplateMap = getPathTemplateMapForSecondaryDS(dataStore.getId(), paths); + pathSnapshotMap = getPathSnapshotMapForSecondaryDS(dataStore.getId(), paths); + pathVolumeMap = getPathVolumeMapForSecondaryDS(dataStore.getId(), paths); + } else { + pathTemplateMap = getPathTemplateMapForPrimaryDS(dataStore.getId(), paths); + pathSnapshotMap = getPathSnapshotMapForPrimaryDS(dataStore.getId(), paths, absPaths); + pathVolumeMap = getPathVolumeMapForPrimaryDS(dataStore.getId(), paths); + } + + for (int i = 0; i < paths.size(); i++) { + DataStoreObjectResponse response = new DataStoreObjectResponse( + answer.getNames().get(i), + answer.getIsDirs().get(i), + answer.getSizes().get(i), + new Date(answer.getLastModified().get(i))); + String filePath = paths.get(i); + if (pathTemplateMap.get(filePath) != null) { + response.setTemplateId(pathTemplateMap.get(filePath).getUuid()); + response.setFormat(pathTemplateMap.get(filePath).getFormat().toString()); + } + if (pathSnapshotMap.get(filePath) != null) { + response.setSnapshotId(pathSnapshotMap.get(filePath).getUuid()); + } + if (pathVolumeMap.get(filePath) != null) { + response.setVolumeId(pathVolumeMap.get(filePath).getUuid()); + } + responses.add(response); + } + + ListResponse listResponse = new ListResponse<>(); + listResponse.setResponses(responses, answer.getCount()); + return listResponse; + } + + List getFormattedPaths(List paths) { + List formattedPaths = new ArrayList<>(); + for (String path : paths) { + String normalizedPath = Path.of(path).normalize().toString(); + if (normalizedPath.startsWith("/")) { + formattedPaths.add(normalizedPath.substring(1)); + } else { + formattedPaths.add(normalizedPath); + } + } + return formattedPaths; + } + + Map getPathTemplateMapForSecondaryDS(Long dataStoreId, List paths) { + Map pathTemplateMap = new HashMap<>(); + List templateList = templateDataStoreDao.listByStoreIdAndInstallPaths(dataStoreId, paths); + if (!CollectionUtils.isEmpty(templateList)) { + List templates = templateDao.listByIds(templateList.stream().map(TemplateDataStoreVO::getTemplateId).collect(Collectors.toList())); + + Map templateMap = templates.stream().collect( + Collectors.toMap(VMTemplateVO::getId, template -> template)); + + for (TemplateDataStoreVO templateDataStore : templateList) { + pathTemplateMap.put(templateDataStore.getInstallPath(), + templateMap.get(templateDataStore.getTemplateId())); + } + } + return pathTemplateMap; + } + + Map getPathSnapshotMapForSecondaryDS(Long dataStoreId, List paths) { + Map snapshotPathMap = new HashMap<>(); + List snapshotDataStoreList = snapshotDataStoreDao.listByStoreAndInstallPaths(dataStoreId, DataStoreRole.Image, paths); + if (!CollectionUtils.isEmpty(snapshotDataStoreList)) { + List snapshots = snapshotDao.listByIds( + snapshotDataStoreList.stream().map(SnapshotDataStoreVO::getSnapshotId).toArray()); + + Map snapshotMap = snapshots.stream().collect( + Collectors.toMap(Snapshot::getId, snapshot -> snapshot)); + + for (SnapshotDataStoreVO snapshotDataStore : snapshotDataStoreList) { + snapshotPathMap.put(snapshotDataStore.getInstallPath(), snapshotMap.get(snapshotDataStore.getSnapshotId())); + } + } + + return snapshotPathMap; + } + + Map getPathTemplateMapForPrimaryDS(Long dataStoreId, List paths) { + Map pathTemplateMap = new HashMap<>(); + // get a map of paths without extension to path. We do this because extension is not saved in database for xen server. + Map pathWithoutExtensionMap = new HashMap<>(); + for (String path : paths) { + if (path.contains(".")) { + String pathWithoutExtension = path.substring(0, path.lastIndexOf(".")); + pathWithoutExtensionMap.put(pathWithoutExtension, path); + } + } + List pathList = Stream.concat(paths.stream(), pathWithoutExtensionMap.keySet().stream()).collect(Collectors.toList()); + List templateStoragePoolList = templatePoolDao.listByPoolIdAndInstallPath(dataStoreId, pathList); + if (!CollectionUtils.isEmpty(templateStoragePoolList)) { + List templates = templateDao.listByIds + (templateStoragePoolList.stream().map(VMTemplateStoragePoolVO::getTemplateId).collect(Collectors.toList())); + + Map templateMap = templates.stream().collect( + Collectors.toMap(VMTemplateVO::getId, template -> template)); + + for (VMTemplateStoragePoolVO templatePool : templateStoragePoolList) { + pathTemplateMap.put(templatePool.getInstallPath(), templateMap.get(templatePool.getTemplateId())); + if (pathWithoutExtensionMap.get(templatePool.getInstallPath()) != null) { + pathTemplateMap.put(pathWithoutExtensionMap.get(templatePool.getInstallPath()), templateMap.get(templatePool.getTemplateId())); + } + } + } + return pathTemplateMap; + } + + Map getPathSnapshotMapForPrimaryDS(Long dataStoreId, List paths, + List absPaths) { + Map snapshotPathMap = new HashMap<>(); + // get a map of paths without extension to path. We do this because extension is not saved in database for xen server. + Map absPathWithoutExtensionMap = new HashMap<>(); + for (String path : absPaths) { + if (path.contains(".")) { + String pathWithoutExtension = path.substring(0, path.lastIndexOf(".")); + absPathWithoutExtensionMap.put(pathWithoutExtension, path); + } + } + List absPathList = Stream.concat(absPaths.stream(), absPathWithoutExtensionMap.keySet().stream()).collect(Collectors.toList()); + // For primary dataStore, we query using absolutePaths + List snapshotDataStoreList = snapshotDataStoreDao.listByStoreAndInstallPaths(dataStoreId, DataStoreRole.Primary, absPathList); + if (!CollectionUtils.isEmpty(snapshotDataStoreList)) { + List snapshots = snapshotDao.listByIds(snapshotDataStoreList.stream().map(SnapshotDataStoreVO::getSnapshotId).toArray()); + + Map snapshotMap = snapshots.stream().collect( + Collectors.toMap(Snapshot::getId, snapshot -> snapshot)); + + // In case of primary data store, absolute path is stored in database. + // We use this map to create a mapping between relative path and absolute path + // which is used to create a mapping between relative path and snapshot. + Map absolutePathPathMap = new HashMap<>(); + for (int i = 0; i < paths.size(); i++) { + absolutePathPathMap.put(absPaths.get(i), paths.get(i)); + } + + for (SnapshotDataStoreVO snapshotDataStore : snapshotDataStoreList) { + snapshotPathMap.put(absolutePathPathMap.get(snapshotDataStore.getInstallPath()), + snapshotMap.get(snapshotDataStore.getSnapshotId())); + + if (absPathWithoutExtensionMap.get(snapshotDataStore.getInstallPath()) != null) { + snapshotPathMap.put( + absolutePathPathMap.get( + absPathWithoutExtensionMap.get( + snapshotDataStore.getInstallPath() + )), snapshotMap.get(snapshotDataStore.getSnapshotId())); + } + } + } + + return snapshotPathMap; + } + + Map getPathVolumeMapForPrimaryDS(Long dataStoreId, List paths) { + Map volumePathMap = new HashMap<>(); + + // get a map of paths without extension to path. We do this because extension is not saved in database for xen server. + Map pathWithoutExtensionMap = new HashMap<>(); + for (String path : paths) { + if (path.contains(".")) { + String pathWithoutExtension = path.substring(0, path.lastIndexOf(".")); + pathWithoutExtensionMap.put(pathWithoutExtension, path); + } + } + List pathList = Stream.concat(paths.stream(), pathWithoutExtensionMap.keySet().stream()).collect(Collectors.toList()); + List volumeList = volumeDao.listByPoolIdAndPaths(dataStoreId, pathList); + if (!CollectionUtils.isEmpty(volumeList)) { + for (VolumeVO volume : volumeList) { + volumePathMap.put(volume.getPath(), volume); + if (pathWithoutExtensionMap.get(volume.getPath()) != null) { + volumePathMap.put(pathWithoutExtensionMap.get(volume.getPath()), volume); + } + } + } + return volumePathMap; + } + + Map getPathVolumeMapForSecondaryDS(Long dataStoreId, List paths) { + Map volumePathMap = new HashMap<>(); + List volumeList = volumeDataStoreDao.listByStoreIdAndInstallPaths(dataStoreId, paths); + if (!CollectionUtils.isEmpty(volumeList)) { + List volumeIdList = volumeList.stream().map(VolumeDataStoreVO::getVolumeId).collect(Collectors.toList()); + List volumeVOS = volumeDao.listByIds(volumeIdList); + Map volumeMap = volumeVOS.stream().collect(Collectors.toMap(VolumeVO::getId, volume -> volume)); + + for (VolumeDataStoreVO volumeDataStore : volumeList) { + volumePathMap.put(volumeDataStore.getInstallPath(), volumeMap.get(volumeDataStore.getVolumeId())); + } + } + return volumePathMap; + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index 3443e2403dd..ca3a77b8e89 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -337,6 +337,8 @@ + + diff --git a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java index 08ba0955f49..764d9437fa9 100644 --- a/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java +++ b/server/src/test/java/com/cloud/api/query/QueryManagerImplTest.java @@ -225,6 +225,7 @@ public class QueryManagerImplTest { Assert.assertEquals(expectedAccessMethods, options.get(VNF.AccessDetail.ACCESS_METHODS.name().toLowerCase())); } + @Test public void applyPublicTemplateRestrictionsTestDoesNotApplyRestrictionsWhenCallerIsRootAdmin() { Mockito.when(accountMock.getType()).thenReturn(Account.Type.ADMIN); diff --git a/server/src/test/java/org/apache/cloudstack/storage/browser/StorageBrowserImplTest.java b/server/src/test/java/org/apache/cloudstack/storage/browser/StorageBrowserImplTest.java new file mode 100644 index 00000000000..7ba9f697b46 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/storage/browser/StorageBrowserImplTest.java @@ -0,0 +1,459 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.cloudstack.storage.browser; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.UnsupportedAnswer; +import com.cloud.api.query.dao.ImageStoreJoinDao; +import com.cloud.api.query.vo.ImageStoreJoinVO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.SnapshotVO; +import com.cloud.storage.Storage; +import com.cloud.storage.VMTemplateStoragePoolVO; +import com.cloud.storage.VMTemplateVO; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.SnapshotDao; +import com.cloud.storage.dao.VMTemplateDao; +import com.cloud.storage.dao.VMTemplatePoolDao; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.cloudstack.api.command.admin.storage.DownloadImageStoreObjectCmd; +import org.apache.cloudstack.api.command.admin.storage.ListImageStoreObjectsCmd; +import org.apache.cloudstack.api.command.admin.storage.ListStoragePoolObjectsCmd; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPoint; +import org.apache.cloudstack.engine.subsystem.api.storage.EndPointSelector; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsAnswer; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.VolumeDataStoreDao; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class StorageBrowserImplTest { + + @Mock + ImageStoreJoinDao imageStoreJoinDao; + + @Mock + DataStoreManager dataStoreMgr; + + @Mock + TemplateDataStoreDao templateDataStoreDao; + + @Mock + SnapshotDataStoreDao snapshotDataStoreDao; + + @Mock + SnapshotDao snapshotDao; + + @Mock + EndPointSelector endPointSelector; + + @Mock + VMTemplatePoolDao templatePoolDao; + + @Mock + VMTemplateDao templateDao; + + @Mock + VolumeDao volumeDao; + + @Mock + VolumeDataStoreDao volumeDataStoreDao; + + @InjectMocks + @Spy + private StorageBrowserImpl storageBrowser; + + private AutoCloseable closeable; + + @Before + public void setUp() throws Exception { + closeable = MockitoAnnotations.openMocks(this); + } + + @After + public void tearDown() throws Exception { + closeable.close(); + } + + @Test + public void testGetPathVolumeMapForPrimaryDSNoVolumes() { + List paths = List.of("volume1", "volume2"); + Mockito.when(volumeDao.listByPoolIdAndPaths(1, paths)).thenReturn(null); + Map result = storageBrowser.getPathVolumeMapForPrimaryDS(1L, paths); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testGetPathVolumeMapForPrimaryDS() { + List paths = List.of("volume1", "volume2"); + VolumeVO volume = Mockito.mock(VolumeVO.class); + Mockito.when(volume.getPath()).thenReturn("volume1"); + + Mockito.when(volumeDao.listByPoolIdAndPaths(1, paths)).thenReturn(List.of(volume)); + Map result = storageBrowser.getPathVolumeMapForPrimaryDS(1L, paths); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(volume, result.get("volume1")); + } + + @Test + public void testGetPathTemplateMapForPrimaryDSNoTemplates() { + List paths = List.of("template1", "template2"); + Mockito.when(templatePoolDao.listByPoolIdAndInstallPath(1L, paths)).thenReturn(null); + Map result = storageBrowser.getPathTemplateMapForPrimaryDS(1L, paths); + Assert.assertTrue(result.isEmpty()); + } + + @Test + public void testGetPathTemplateMapForPrimaryDS() { + List paths = List.of("template1", "template2"); + VMTemplateStoragePoolVO templatePool = Mockito.mock(VMTemplateStoragePoolVO.class); + Mockito.when(templatePool.getTemplateId()).thenReturn(5L); + Mockito.when(templatePool.getInstallPath()).thenReturn("template1"); + + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Mockito.when(template.getId()).thenReturn(5L); + + Mockito.when(templateDao.listByIds(List.of(5L))).thenReturn(List.of(template)); + + List templateStoragePoolList = List.of(templatePool); + Mockito.when(templatePoolDao.listByPoolIdAndInstallPath(1L, paths)).thenReturn(templateStoragePoolList); + Map result = storageBrowser.getPathTemplateMapForPrimaryDS(1L, paths); + Assert.assertEquals(1L, result.size()); + Assert.assertEquals(template, result.get("template1")); + } + + @Test + public void testGetFormattedPaths() { + List paths = List.of("/path/to/file", "/path/to/file/", "path/to/file", "path/to/file/"); + List expectedFormattedPaths = List.of("path/to/file", "path/to/file", "path/to/file", "path/to/file"); + + List formattedPaths = storageBrowser.getFormattedPaths(paths); + + Assert.assertEquals(expectedFormattedPaths, formattedPaths); + } + + @Test + public void testListImageStore() { + ListImageStoreObjectsCmd cmd = Mockito.mock(ListImageStoreObjectsCmd.class); + Long imageStoreId = 1L; + String path = "path/to/image/store"; + Mockito.when(cmd.getStoreId()).thenReturn(imageStoreId); + Mockito.when(cmd.getPath()).thenReturn(path); + Mockito.when(cmd.getStartIndex()).thenReturn(0L); + Mockito.when(cmd.getPageSizeVal()).thenReturn(10L); + + ImageStoreJoinVO imageStore = Mockito.mock(ImageStoreJoinVO.class); + Mockito.when(imageStoreJoinDao.findById(imageStoreId)).thenReturn(imageStore); + + DataStore dataStore = Mockito.mock(DataStore.class); + Mockito.when(dataStoreMgr.getDataStore(imageStoreId, imageStore.getRole())).thenReturn(dataStore); + + ListDataStoreObjectsAnswer answer = Mockito.mock(ListDataStoreObjectsAnswer.class); + Mockito.doReturn(answer).when(storageBrowser).listObjectsInStore(dataStore, path, 0, 10); + + ListResponse response = storageBrowser.listImageStoreObjects(cmd); + + Assert.assertNotNull(response); + } + + @Test + public void testListPrimaryStore() { + ListStoragePoolObjectsCmd cmd = Mockito.mock(ListStoragePoolObjectsCmd.class); + Long storeId = 1L; + String path = "path/to/primary/store"; + Mockito.when(cmd.getStoreId()).thenReturn(storeId); + Mockito.when(cmd.getPath()).thenReturn(path); + Mockito.when(cmd.getStartIndex()).thenReturn(0L); + Mockito.when(cmd.getPageSizeVal()).thenReturn(10L); + + DataStore dataStore = Mockito.mock(DataStore.class); + Mockito.when(dataStoreMgr.getDataStore(storeId, DataStoreRole.Primary)).thenReturn(dataStore); + + ListDataStoreObjectsAnswer answer = Mockito.mock(ListDataStoreObjectsAnswer.class); + Mockito.doReturn(answer).when(storageBrowser).listObjectsInStore(dataStore, path, 0, 10); + + ListResponse response = storageBrowser.listPrimaryStoreObjects(cmd); + + Assert.assertNotNull(response); + Assert.assertEquals(answer.getPaths().size(), response.getResponses().size()); + } + + @Test + public void testGetCommands() { + List> expectedCmdList = new ArrayList<>(); + expectedCmdList.add(ListImageStoreObjectsCmd.class); + expectedCmdList.add(ListStoragePoolObjectsCmd.class); + expectedCmdList.add(DownloadImageStoreObjectCmd.class); + + List> cmdList = storageBrowser.getCommands(); + + Assert.assertNotNull(cmdList); + Assert.assertEquals(expectedCmdList, cmdList); + } + + @Test + public void testListObjectsInStoreNoEndpoint() { + DataStore dataStore = Mockito.mock(DataStore.class); + String path = "path/to/store"; + int startIndex = 0; + int pageSize = 10; + + Mockito.when(endPointSelector.select(dataStore)).thenReturn(null); + + try { + storageBrowser.listObjectsInStore(dataStore, path, startIndex, pageSize); + } catch (CloudRuntimeException exception) { + Assert.assertEquals("No remote endpoint to send command", exception.getMessage()); + } + } + + @Test + public void testListObjectsInStoreBadCommand() { + DataStore dataStore = Mockito.mock(DataStore.class); + String path = "path/to/store"; + int startIndex = 0; + int pageSize = 10; + + EndPoint ep = Mockito.mock(EndPoint.class); + + Mockito.when(endPointSelector.select(dataStore)).thenReturn(ep); + Answer answer = Mockito.mock(UnsupportedAnswer.class); + Mockito.when(ep.sendMessage(Mockito.any())).thenReturn(answer); + + try { + storageBrowser.listObjectsInStore(dataStore, path, startIndex, pageSize); + } catch (CloudRuntimeException exception) { + Assert.assertEquals("Failed to list datastore objects", exception.getMessage()); + } + } + + @Test + public void testListObjectsInStorePathDoesNotExist() { + DataStore dataStore = Mockito.mock(DataStore.class); + String path = "path/to/store"; + int startIndex = 0; + int pageSize = 10; + + EndPoint ep = Mockito.mock(EndPoint.class); + + Mockito.when(endPointSelector.select(dataStore)).thenReturn(ep); + ListDataStoreObjectsAnswer answer = Mockito.mock(ListDataStoreObjectsAnswer.class); + Mockito.when(ep.sendMessage(Mockito.any())).thenReturn(answer); + Mockito.when(answer.getResult()).thenReturn(true); + Mockito.when(answer.isPathExists()).thenReturn(false); + Mockito.when(dataStore.getUuid()).thenReturn("uuid"); + + try { + storageBrowser.listObjectsInStore(dataStore, path, startIndex, pageSize); + } catch (IllegalArgumentException exception) { + Assert.assertEquals("Path " + path + " doesn't exist in store: " + dataStore.getUuid(), exception.getMessage()); + } + } + + @Test + public void testListObjectsInStore() { + DataStore dataStore = Mockito.mock(DataStore.class); + String path = "path/to/store"; + int startIndex = 0; + int pageSize = 10; + + EndPoint ep = Mockito.mock(EndPoint.class); + + Mockito.when(endPointSelector.select(dataStore)).thenReturn(ep); + ListDataStoreObjectsAnswer answer = Mockito.mock(ListDataStoreObjectsAnswer.class); + Mockito.when(ep.sendMessage(Mockito.any())).thenReturn(answer); + Mockito.when(answer.getResult()).thenReturn(true); + Mockito.when(answer.isPathExists()).thenReturn(true); + + Assert.assertEquals(answer, storageBrowser.listObjectsInStore(dataStore, path, startIndex, pageSize)); + } + + @Test + public void testGetPathTemplateMapForSecondaryDS() { + long dataStoreId = 1L; + List paths = List.of("/path1", "/path2"); + + TemplateDataStoreVO templateDataStore1 = Mockito.mock(TemplateDataStoreVO.class); + Mockito.when(templateDataStore1.getInstallPath()).thenReturn("/path1"); + Mockito.when(templateDataStore1.getTemplateId()).thenReturn(1L); + + TemplateDataStoreVO templateDataStore2 = Mockito.mock(TemplateDataStoreVO.class); + Mockito.when(templateDataStore2.getInstallPath()).thenReturn("/path2"); + Mockito.when(templateDataStore2.getTemplateId()).thenReturn(2L); + + List templateList = List.of(templateDataStore1, templateDataStore2); + + VMTemplateVO template1 = Mockito.mock(VMTemplateVO.class); + Mockito.when(template1.getId()).thenReturn(1L); + + VMTemplateVO template2 = Mockito.mock(VMTemplateVO.class); + Mockito.when(template2.getId()).thenReturn(2L); + + List templates = List.of(template1, template2); + + Mockito.when(templateDataStoreDao.listByStoreIdAndInstallPaths(dataStoreId, paths)).thenReturn(templateList); + Mockito.when(templateDao.listByIds(List.of(1L, 2L))).thenReturn(templates); + + Map expectedPathTemplateMap = new HashMap<>(); + expectedPathTemplateMap.put("/path1", template1); + expectedPathTemplateMap.put("/path2", template2); + + Map actualPathTemplateMap = storageBrowser.getPathTemplateMapForSecondaryDS(dataStoreId, paths); + + Assert.assertEquals(expectedPathTemplateMap, actualPathTemplateMap); + } + + @Test + public void testGetPathSnapshotMapForSecondaryDS() { + long dataStoreId = 1L; + List paths = List.of("/path1", "/path2"); + + SnapshotDataStoreVO snapshotDataStore1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(snapshotDataStore1.getInstallPath()).thenReturn("/path1"); + Mockito.when(snapshotDataStore1.getSnapshotId()).thenReturn(1L); + + SnapshotDataStoreVO snapshotDataStore2 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(snapshotDataStore2.getInstallPath()).thenReturn("/path2"); + Mockito.when(snapshotDataStore2.getSnapshotId()).thenReturn(2L); + + List snapshotDataStoreList = List.of(snapshotDataStore1, snapshotDataStore2); + + SnapshotVO snapshot1 = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshot1.getId()).thenReturn(1L); + + SnapshotVO snapshot2 = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshot2.getId()).thenReturn(2L); + + List snapshots = List.of(snapshot1, snapshot2); + + Mockito.when(snapshotDataStoreDao.listByStoreAndInstallPaths(dataStoreId, DataStoreRole.Image, paths)).thenReturn(snapshotDataStoreList); + Mockito.when(snapshotDao.listByIds(new Long[]{1L, 2L})).thenReturn(snapshots); + + Map expectedSnapshotPathMap = new HashMap<>(); + expectedSnapshotPathMap.put("/path1", snapshot1); + expectedSnapshotPathMap.put("/path2", snapshot2); + + Map snapshotPathMap = storageBrowser.getPathSnapshotMapForSecondaryDS(dataStoreId, paths); + + Assert.assertEquals(expectedSnapshotPathMap, snapshotPathMap); + } + + @Test + public void testGetPathSnapshotMapForPrimaryDS() { + long dataStoreId = 1L; + List paths = List.of("path1", "path2"); + List absPaths = List.of("/mnt/path1", "/mnt/path2"); + + SnapshotDataStoreVO snapshotDataStore1 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(snapshotDataStore1.getInstallPath()).thenReturn("/mnt/path1"); + Mockito.when(snapshotDataStore1.getSnapshotId()).thenReturn(1L); + + SnapshotDataStoreVO snapshotDataStore2 = Mockito.mock(SnapshotDataStoreVO.class); + Mockito.when(snapshotDataStore2.getInstallPath()).thenReturn("/mnt/path2"); + Mockito.when(snapshotDataStore2.getSnapshotId()).thenReturn(2L); + + List snapshotDataStoreList = List.of(snapshotDataStore1, snapshotDataStore2); + + SnapshotVO snapshot1 = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshot1.getId()).thenReturn(1L); + + SnapshotVO snapshot2 = Mockito.mock(SnapshotVO.class); + Mockito.when(snapshot2.getId()).thenReturn(2L); + + List snapshots = List.of(snapshot1, snapshot2); + + Mockito.when(snapshotDataStoreDao.listByStoreAndInstallPaths(dataStoreId, DataStoreRole.Primary, absPaths)).thenReturn(snapshotDataStoreList); + Mockito.when(snapshotDao.listByIds(new Long[]{1L, 2L})).thenReturn(snapshots); + + Map expectedSnapshotPathMap = new HashMap<>(); + expectedSnapshotPathMap.put("path1", snapshot1); + expectedSnapshotPathMap.put("path2", snapshot2); + + Map snapshotPathMap = storageBrowser.getPathSnapshotMapForPrimaryDS(dataStoreId, paths, absPaths); + + Assert.assertEquals(expectedSnapshotPathMap, snapshotPathMap); + } + + @Test + public void testGetResponse() { + ListDataStoreObjectsAnswer answer = Mockito.mock(ListDataStoreObjectsAnswer.class); + Mockito.when(answer.getPaths()).thenReturn(List.of("/path1", "/path2")); + Mockito.when(answer.getAbsPaths()).thenReturn(List.of("/path1", "/path2")); + Mockito.when(answer.getNames()).thenReturn(List.of("name1", "name2")); + Mockito.when(answer.getIsDirs()).thenReturn(List.of(true, false)); + Mockito.when(answer.getSizes()).thenReturn(List.of(100L, 200L)); + Mockito.when(answer.getLastModified()).thenReturn(List.of((new Date()).getTime(), (new Date()).getTime())); + + List paths = List.of("path1", "path2"); + List absPaths = List.of("/path1", "/path2"); + + Map pathSnapshotMap = new HashMap<>(); + pathSnapshotMap.put("path1", Mockito.mock(SnapshotVO.class)); + + VMTemplateVO template = Mockito.mock(VMTemplateVO.class); + Map pathTemplateMap = new HashMap<>(); + pathTemplateMap.put("path2", template); + Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.ISO); + + Map pathVolumeMap = new HashMap<>(); + pathVolumeMap.put("path1", Mockito.mock(VolumeVO.class)); + + DataStore dataStore = Mockito.mock(DataStore.class); + + Mockito.when(dataStore.getRole()).thenReturn(DataStoreRole.Primary); + Mockito.when(dataStore.getId()).thenReturn(1L); + Mockito.doReturn(pathTemplateMap).when(storageBrowser).getPathTemplateMapForPrimaryDS(1L, paths); + Mockito.doReturn(pathSnapshotMap).when(storageBrowser).getPathSnapshotMapForPrimaryDS(1L, paths, absPaths); + Mockito.doReturn(pathVolumeMap).when(storageBrowser).getPathVolumeMapForPrimaryDS(1L, paths); + + ListResponse response = storageBrowser.getResponse(dataStore, answer); + + Assert.assertEquals(2, response.getResponses().size()); + Assert.assertEquals("name1", response.getResponses().get(0).getName()); + Assert.assertEquals(true, response.getResponses().get(0).isDirectory()); + Assert.assertEquals(100L, response.getResponses().get(0).getSize()); + Assert.assertNotNull(response.getResponses().get(0).getLastUpdated()); + + Assert.assertEquals("name2", response.getResponses().get(1).getName()); + Assert.assertEquals(false, response.getResponses().get(1).isDirectory()); + Assert.assertEquals(200L, response.getResponses().get(1).getSize()); + Assert.assertNotNull(response.getResponses().get(1).getLastUpdated()); + } +} diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 4883bb77dca..5d76393d81e 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -68,6 +68,7 @@ import org.apache.cloudstack.storage.command.TemplateOrVolumePostUploadCommand; import org.apache.cloudstack.storage.command.UploadStatusAnswer; import org.apache.cloudstack.storage.command.UploadStatusAnswer.UploadStatus; import org.apache.cloudstack.storage.command.UploadStatusCommand; +import org.apache.cloudstack.storage.command.browser.ListDataStoreObjectsCommand; import org.apache.cloudstack.storage.configdrive.ConfigDrive; import org.apache.cloudstack.storage.configdrive.ConfigDriveBuilder; import org.apache.cloudstack.storage.template.DownloadManager; @@ -322,6 +323,8 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return execute((CreateDatadiskTemplateCommand)cmd); } else if (cmd instanceof MoveVolumeCommand) { return execute((MoveVolumeCommand)cmd); + } else if (cmd instanceof ListDataStoreObjectsCommand) { + return execute((ListDataStoreObjectsCommand)cmd); } else if (cmd instanceof QuerySnapshotZoneCopyCommand) { return execute((QuerySnapshotZoneCopyCommand)cmd); } else { @@ -329,6 +332,10 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } + private Answer execute(ListDataStoreObjectsCommand cmd) { + return listFilesAtPath(getRootDir(cmd.getStore().getUrl(), _nfsVersion), cmd.getPath(), cmd.getStartIndex(), cmd.getPageSize()); + } + private Answer execute(HandleConfigDriveIsoCommand cmd) { if (cmd.isCreate()) { if (cmd.getIsoData() == null) { diff --git a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java index 8e3f0b211ff..5c589b6e35c 100644 --- a/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java +++ b/services/secondary-storage/server/src/main/java/org/apache/cloudstack/storage/template/UploadManagerImpl.java @@ -281,6 +281,14 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE); } + File file = new File("/mnt/SecStorage/" + cmd.getParent() + File.separator + cmd.getInstallPath()); + // Return error if the file does not exist or is a directory + if (!file.exists() || file.isDirectory()) { + String errorString = "Error in finding the file " + file.getAbsolutePath(); + s_logger.error(errorString); + return new CreateEntityDownloadURLAnswer(errorString, CreateEntityDownloadURLAnswer.RESULT_FAILURE); + } + // Create a random file under the directory for security reasons. String uuid = cmd.getExtractLinkUUID(); // Create a symbolic link from the actual directory to the template location. The entity would be directly visible under /var/www/html/userdata/cmd.getInstallPath(); @@ -302,7 +310,7 @@ public class UploadManagerImpl extends ManagerBase implements UploadManager { public Answer handleDeleteEntityDownloadURLCommand(DeleteEntityDownloadURLCommand cmd) { //Delete the soft link. Example path = volumes/8/74eeb2c6-8ab1-4357-841f-2e9d06d1f360.vhd - s_logger.warn("handleDeleteEntityDownloadURLCommand Path:" + cmd.getPath() + " Type:" + cmd.getType().toString()); + s_logger.warn("handleDeleteEntityDownloadURLCommand Path:" + cmd.getPath() + " Type:" + (cmd.getType() != null ? cmd.getType().toString(): "")); String path = cmd.getPath(); Script command = new Script("/bin/bash", s_logger); command.add("-c"); diff --git a/test/integration/smoke/test_image_store_object_migration.py b/test/integration/smoke/test_image_store_object_migration.py new file mode 100644 index 00000000000..de504aec810 --- /dev/null +++ b/test/integration/smoke/test_image_store_object_migration.py @@ -0,0 +1,233 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" Image store object migration test +""" +import logging + +#Import Local Modules +from marvin.cloudstackException import * +from marvin.cloudstackAPI import * +from marvin.codes import FAILED +from marvin.cloudstackTestCase import cloudstackTestCase +import unittest +from marvin.cloudstackAPI import listZones +from marvin.lib.utils import random_gen, cleanup_resources +from marvin.lib.base import (Template, + ImageStore) +from marvin.lib.common import (get_domain, + get_zone, + get_template) +from nose.plugins.attrib import attr +import urllib.request, urllib.parse, urllib.error +#Import System modules +import time +from marvin.cloudstackAPI import (createTemplate, listOsTypes) + +_multiprocess_shared_ = True + +class TestImageStoreObjectMigration(cloudstackTestCase): + """Test Image Store Object migration + """ + def setUp(self): + self.testClient = super(TestImageStoreObjectMigration, self).getClsTestClient() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + + self.services = self.testClient.getParsedTestDataConfig() + self.unsupportedHypervisor = False + self.hypervisor = self.testClient.getHypervisorInfo() + if self.hypervisor.lower() in ['lxc']: + # Template creation from root volume is not supported in LXC + self.unsupportedHypervisor = True + return + + # Get Zone, Domain and templates + self.domain = get_domain(self.apiclient) + self.zone = get_zone(self.apiclient, self.testClient.getZoneForTests()) + + if "kvm" in self.hypervisor.lower(): + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template.checksum = "{SHA-1}" + "6952e58f39b470bd166ace11ffd20bf479bed936" + self.test_template.hypervisor = self.hypervisor + self.test_template.zoneid = self.zone.id + self.test_template.name = 'test sha-2333' + self.test_template.displaytext = 'test sha-1' + self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-kvm.qcow2.bz2" + self.test_template.format = "QCOW2" + self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "88c60fd500ce7ced985cf845df0db9da" + self.sha256 = "bc4cc040bbab843000fab78db6cb4a33f3a06ae1ced2cf563d36b38c7fee3049" + + if "vmware" in self.hypervisor.lower(): + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template.checksum = "{SHA-1}" + "8b82224fd3c6429b6914f32d8339e650770c7526" + self.test_template.hypervisor = self.hypervisor + self.test_template.zoneid = self.zone.id + self.test_template.name = 'test sha-2333' + self.test_template.displaytext = 'test sha-1' + self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-vmware.ova" + self.test_template.format = "OVA" + self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "b4e8bff3882b23175974e692533b4381" + self.sha256 = "e1dffca3c3ab545a753cb42d838a341624cf25841d1bcf3d1e45556c9fce7cf3" + + if "xen" in self.hypervisor.lower(): + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template = registerTemplate.registerTemplateCmd() + self.test_template.checksum = "{SHA-1}" + "80af2c18f96e94273188808c3d56e561a1cda717" + self.test_template.hypervisor = self.hypervisor + self.test_template.zoneid = self.zone.id + self.test_template.name = 'test sha-2333' + self.test_template.displaytext = 'test sha-1' + self.test_template.url = "http://dl.openvm.eu/cloudstack/macchinina/x86_64/macchinina-xen.vhd.bz2" + self.test_template.format = "VHD" + self.test_template.ostypeid = self.getOsType("Other Linux (64-bit)") + self.md5 = "1662bbf224e41bb62b1dee043d785731" + self.sha256 = "80fba5a7a83842ec4e5f67cc6755d61d4fca46ae170d59b0c6ed47ebf7162722" + + if self.unsupportedHypervisor: + self.skipTest("Skipping test because unsupported hypervisor\ + %s" % self.hypervisor) + return + + def tearDown(self): + try: + # Clean up the created templates + for temp in self.cleanup: + cmd = deleteTemplate.deleteTemplateCmd() + cmd.id = temp.id + cmd.zoneid = self.zone.id + self.apiclient.deleteTemplate(cmd) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return + + @attr(tags = ["advanced", "basic", "smoke"], required_hardware="true") + def test_01_browser_migrate_template(self): + """ + Test storage browser and template migration to another secondary storage + """ + template = self.registerTemplate(self.test_template) + self.download(self.apiclient, template.id) + + list_template_response=Template.list( + self.apiclient, + id=template.id, + templatefilter="all", + zoneid=self.zone.id) + + datastoreid = list_template_response[0].downloaddetails[0].datastoreId + + qresultset = self.dbclient.execute( + "select account_id, id from vm_template where uuid = '%s';" + % template.id + ) + + account_id = qresultset[0][0] + template_id = qresultset[0][1] + + originalSecondaryStore = ImageStore({"id": datastoreid}) + + storeObjects = originalSecondaryStore.listObjects(self.apiclient, path="template/tmpl/" + str(account_id) + "/" + str(template_id)) + + self.assertEqual(len(storeObjects), 2, "Check template is uploaded on secondary storage") + + # Migrate template to another secondary storage + secondaryStores = ImageStore.list(self.apiclient, zoneid=self.zone.id) + + if len(secondaryStores) < 2: + self.skipTest("Only one secondary storage available hence skipping") + + for store in secondaryStores: + if store.id != datastoreid: + destSecondaryStore = ImageStore({"id": store.id}) + break + + originalSecondaryStore.migrateResources(self.apiclient, destSecondaryStore.id, templateIdList=[template.id]) + + try: + originalSecondaryStore.listObjects(self.apiclient, path="template/tmpl/" + str(account_id) + "/" + str(template_id)) + except Exception as exc: + self.assertTrue("template/tmpl/" + str(account_id) + "/" + str(template_id) + " doesn't exist in store" in str(exc), + "Check template is deleted from original secondary storage") + else: + self.fail("Template is not deleted from original secondary storage") + + storeObjects = destSecondaryStore.listObjects(self.apiclient, path="template/tmpl/" + str(account_id) + "/" + str(template_id)) + + self.assertEqual(len(storeObjects), 2, "Check template is uploaded on destination secondary storage") + + def registerTemplate(self, cmd): + temp = self.apiclient.registerTemplate(cmd)[0] + if not temp: + self.cleanup.append(temp) + return temp + + def getOsType(self, param): + cmd = listOsTypes.listOsTypesCmd() + cmd.description = param + return self.apiclient.listOsTypes(cmd)[0].id + + def download(self, apiclient, template_id, retries=12, interval=5): + """Check if template download will finish in 1 minute""" + while retries > -1: + time.sleep(interval) + template_response = Template.list( + apiclient, + id=template_id, + zoneid=self.zone.id, + templatefilter='self' + ) + + if isinstance(template_response, list): + template = template_response[0] + if not hasattr(template, 'status') or not template or not template.status: + retries = retries - 1 + continue + + # If template is ready, + # template.status = Download Complete + # Downloading - x% Downloaded + # if Failed + # Error - Any other string + if 'Failed' in template.status: + raise Exception( + "Failed to download template: status - %s" % + template.status) + + elif template.status == 'Download Complete' and template.isready: + return + + elif 'Downloaded' in template.status: + retries = retries - 1 + continue + + elif 'Installing' not in template.status: + if retries >= 0: + retries = retries - 1 + continue + raise Exception( + "Error in downloading template: status - %s" % + template.status) + + else: + retries = retries - 1 + raise Exception("Template download failed exception.") diff --git a/tools/apidoc/gen_toc.py b/tools/apidoc/gen_toc.py index d08918a66bc..d46e544666c 100644 --- a/tools/apidoc/gen_toc.py +++ b/tools/apidoc/gen_toc.py @@ -210,6 +210,7 @@ known_categories = { 'deleteSecondaryStagingStore': 'Image Store', 'listSecondaryStagingStores': 'Image Store', 'updateImageStore': 'Image Store', + 'downloadImageStoreObject': 'Image Store', 'InternalLoadBalancer': 'Internal LB', 'DeploymentPlanners': 'Configuration', 'ObjectStore': 'Image Store', diff --git a/tools/marvin/marvin/lib/base.py b/tools/marvin/marvin/lib/base.py index fd4b39a0ee9..b0fd3198301 100755 --- a/tools/marvin/marvin/lib/base.py +++ b/tools/marvin/marvin/lib/base.py @@ -3504,6 +3504,13 @@ class StoragePool: timeout -= 60 return returnValue + @classmethod + def listObjects(cls, apiclient, path="/"): + cmd = listStoragePoolObjects.listStoragePoolObjectsCmd() + cmd.id = cls.id + cmd.path = path + return apiclient.listStoragePoolObjects(cmd) + class Network: """Manage Network pools""" @@ -4214,6 +4221,20 @@ class ImageStore: cmd.listall = True return (apiclient.listImageStores(cmd)) + def listObjects(self, apiclient, path="/"): + cmd = listImageStoreObjects.listImageStoreObjectsCmd() + cmd.id = self.id + cmd.path = path + return apiclient.listImageStoreObjects(cmd) + + def migrateResources(self, apiclient, destStoreId, templateIdList=[], snapshotIdList=[]): + cmd = migrateResourceToAnotherSecondaryStorage.migrateResourceToAnotherSecondaryStorageCmd() + cmd.srcpool = self.id + cmd.destpool = destStoreId + cmd.templates = templateIdList + cmd.snapshots = snapshotIdList + return apiclient.migrateResourceToAnotherSecondaryStorage(cmd) + class PhysicalNetwork: """Manage physical network storage""" diff --git a/ui/public/locales/en.json b/ui/public/locales/en.json index e5c61ce4440..2d6cbe374ad 100644 --- a/ui/public/locales/en.json +++ b/ui/public/locales/en.json @@ -42,6 +42,7 @@ "label.acquire.new.ip": "Acquire new IP", "label.acquire.new.secondary.ip": "Acquire new secondary IP", "label.acquiring.ip": "Acquiring IP", +"label.associated.resource": "Associated resource", "label.action": "Action", "label.action.attach.disk": "Attach disk", "label.action.attach.iso": "Attach ISO", @@ -404,6 +405,7 @@ "label.broadcastdomaintype": "Broadcast domain type", "label.broadcasturi": "Broadcast URI", "label.brocade.vcs.address": "Vcs switch address", +"label.browser": "Browser", "label.bucket": "Bucket", "label.by.account": "By account", "label.by.domain": "By domain", @@ -988,6 +990,7 @@ "label.ikepolicy": "IKE policy", "label.ikeversion": "IKE version", "label.images": "Images", +"label.imagestoreid": "Secondary Storage", "label.import.backup.offering": "Import backup offering", "label.import.instance": "Import instance", "label.import.offering": "Import offering", @@ -2884,6 +2887,7 @@ "message.migrate.instance.host.auto.assign": "Host for the instance will be automatically chosen based on the suitability within the same cluster", "message.migrate.instance.to.host": "Please confirm that you want to migrate this instance to another host. When migration is between hosts of different clusters volume(s) of the instance may get migrated to suitable storage pools.", "message.migrate.instance.to.ps": "Please confirm that you want to migrate this instance to another primary storage.", +"message.migrate.resource.to.ss": "Please confirm that you want to migrate this resource to another secondary storage.", "message.migrate.router.confirm": "Please confirm the host you wish to migrate the router to:", "message.migrate.systemvm.confirm": "Please confirm the host you wish to migrate the system VM to:", "message.migrate.volume": "Please confirm that you want to migrate this volume to another primary storage.", @@ -3089,6 +3093,7 @@ "message.success.import.instance": "Successfully imported instance", "message.success.migrate.volume": "Successfully migrated volume", "message.success.migrating": "Migration completed successfully for", +"message.success.migration": "Migration completed successfully", "message.success.move.acl.order": "Successfully moved ACL rule", "message.success.recurring.snapshot": "Successfully recurring snapshots", "message.success.register.iso": "Successfully registered ISO", diff --git a/ui/src/components/view/ImageStoreSelectView.vue b/ui/src/components/view/ImageStoreSelectView.vue new file mode 100644 index 00000000000..13c5a68e5b1 --- /dev/null +++ b/ui/src/components/view/ImageStoreSelectView.vue @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + + + + diff --git a/ui/src/components/view/SearchView.vue b/ui/src/components/view/SearchView.vue index f315667442a..8e5e0088765 100644 --- a/ui/src/components/view/SearchView.vue +++ b/ui/src/components/view/SearchView.vue @@ -279,7 +279,7 @@ export default { if (item === 'groupid' && !('listInstanceGroups' in this.$store.getters.apis)) { return true } - if (['zoneid', 'domainid', 'state', 'level', 'clusterid', 'podid', 'groupid', 'entitytype', 'type'].includes(item)) { + if (['zoneid', 'domainid', 'imagestoreid', 'storageid', 'state', 'level', 'clusterid', 'podid', 'groupid', 'entitytype', 'type'].includes(item)) { type = 'list' } else if (item === 'tags') { type = 'tag' @@ -348,6 +348,8 @@ export default { const promises = [] let zoneIndex = -1 let domainIndex = -1 + let imageStoreIndex = -1 + let storageIndex = -1 let podIndex = -1 let clusterIndex = -1 let groupIndex = -1 @@ -364,6 +366,18 @@ export default { promises.push(await this.fetchDomains(searchKeyword)) } + if (arrayField.includes('imagestoreid')) { + imageStoreIndex = this.fields.findIndex(item => item.name === 'imagestoreid') + this.fields[imageStoreIndex].loading = true + promises.push(await this.fetchImageStores(searchKeyword)) + } + + if (arrayField.includes('storageid')) { + storageIndex = this.fields.findIndex(item => item.name === 'storageid') + this.fields[storageIndex].loading = true + promises.push(await this.fetchStoragePools(searchKeyword)) + } + if (arrayField.includes('podid')) { podIndex = this.fields.findIndex(item => item.name === 'podid') this.fields[podIndex].loading = true @@ -395,6 +409,18 @@ export default { this.fields[domainIndex].opts = this.sortArray(domain[0].data, 'path') } } + if (imageStoreIndex > -1) { + const imageStore = response.filter(item => item.type === 'imagestoreid') + if (imageStore && imageStore.length > 0) { + this.fields[imageStoreIndex].opts = this.sortArray(imageStore[0].data, 'name') + } + } + if (storageIndex > -1) { + const storagePool = response.filter(item => item.type === 'storageid') + if (storagePool && storagePool.length > 0) { + this.fields[storageIndex].opts = this.sortArray(storagePool[0].data, 'name') + } + } if (podIndex > -1) { const pod = response.filter(item => item.type === 'podid') if (pod && pod.length > 0) { @@ -420,6 +446,12 @@ export default { if (domainIndex > -1) { this.fields[domainIndex].loading = false } + if (imageStoreIndex > -1) { + this.fields[imageStoreIndex].loading = false + } + if (storageIndex > -1) { + this.fields[storageIndex].loading = false + } if (podIndex > -1) { this.fields[podIndex].loading = false } @@ -487,6 +519,32 @@ export default { }) }) }, + fetchImageStores (searchKeyword) { + return new Promise((resolve, reject) => { + api('listImageStores', { listAll: true, showicon: true, keyword: searchKeyword }).then(json => { + const imageStore = json.listimagestoresresponse.imagestore + resolve({ + type: 'imagestoreid', + data: imageStore + }) + }).catch(error => { + reject(error.response.headers['x-description']) + }) + }) + }, + fetchStoragePools (searchKeyword) { + return new Promise((resolve, reject) => { + api('listStoragePools', { listAll: true, showicon: true, keyword: searchKeyword }).then(json => { + const storagePool = json.liststoragepoolsresponse.storagepool + resolve({ + type: 'storageid', + data: storagePool + }) + }).catch(error => { + reject(error.response.headers['x-description']) + }) + }) + }, fetchPods (searchKeyword) { return new Promise((resolve, reject) => { api('listPods', { keyword: searchKeyword }).then(json => { diff --git a/ui/src/config/section/image.js b/ui/src/config/section/image.js index 6c848af4bec..792e47f021f 100644 --- a/ui/src/config/section/image.js +++ b/ui/src/config/section/image.js @@ -62,7 +62,14 @@ export default { } return fields }, - searchFilters: ['name', 'zoneid', 'tags'], + searchFilters: () => { + var filters = ['name', 'zoneid', 'tags'] + if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { + filters.push('storageid') + filters.push('imagestoreid') + } + return filters + }, related: [{ name: 'vm', title: 'label.instances', @@ -219,7 +226,14 @@ export default { return fields }, details: ['name', 'id', 'displaytext', 'checksum', 'ostypename', 'size', 'bootable', 'isready', 'directdownload', 'isextractable', 'ispublic', 'isfeatured', 'crosszones', 'account', 'domain', 'created', 'userdatadetails', 'userdatapolicy', 'url'], - searchFilters: ['name', 'zoneid', 'tags'], + searchFilters: () => { + var filters = ['name', 'zoneid', 'tags'] + if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { + filters.push('storageid') + filters.push('imagestoreid') + } + return filters + }, related: [{ name: 'vm', title: 'label.instances', diff --git a/ui/src/config/section/infra/primaryStorages.js b/ui/src/config/section/infra/primaryStorages.js index 2e732a3344a..6f51c7bd0a3 100644 --- a/ui/src/config/section/infra/primaryStorages.js +++ b/ui/src/config/section/infra/primaryStorages.js @@ -39,6 +39,16 @@ export default { name: 'volume', title: 'label.volumes', param: 'storageid' + }, + { + name: 'template', + title: 'label.templates', + param: 'storageid' + }, + { + name: 'iso', + title: 'label.isos', + param: 'storageid' }], resourceType: 'PrimaryStorage', filters: () => { @@ -51,6 +61,10 @@ export default { }, { name: 'settings', component: shallowRef(defineAsyncComponent(() => import('@/components/view/SettingsTab.vue'))) + }, { + name: 'browser', + resourceType: 'PrimaryStorage', + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/StorageBrowser.vue'))) }, { name: 'events', resourceType: 'StoragePool', diff --git a/ui/src/config/section/infra/secondaryStorages.js b/ui/src/config/section/infra/secondaryStorages.js index 93c54dc1d79..774c233c446 100644 --- a/ui/src/config/section/infra/secondaryStorages.js +++ b/ui/src/config/section/infra/secondaryStorages.js @@ -42,12 +42,31 @@ export default { return fields }, resourceType: 'SecondaryStorage', + related: [{ + name: 'template', + title: 'label.templates', + param: 'imagestoreid' + }, + { + name: 'iso', + title: 'label.isos', + param: 'imagestoreid' + }, + { + name: 'snapshot', + title: 'label.snapshots', + param: 'imagestoreid' + }], tabs: [{ name: 'details', component: shallowRef(defineAsyncComponent(() => import('@/components/view/DetailsTab.vue'))) }, { name: 'settings', component: shallowRef(defineAsyncComponent(() => import('@/components/view/SettingsTab.vue'))) + }, { + name: 'browser', + resourceType: 'ImageStore', + component: shallowRef(defineAsyncComponent(() => import('@/views/infra/StorageBrowser.vue'))) }, { name: 'events', resourceType: 'ImageStore', diff --git a/ui/src/config/section/storage.js b/ui/src/config/section/storage.js index 8770f8edc73..3d0223ffd6c 100644 --- a/ui/src/config/section/storage.js +++ b/ui/src/config/section/storage.js @@ -88,7 +88,13 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) } ], - searchFilters: ['name', 'zoneid', 'domainid', 'account', 'state', 'tags'], + searchFilters: () => { + var filters = ['name', 'zoneid', 'domainid', 'account', 'state', 'tags'] + if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { + filters.push('storageid') + } + return filters + }, actions: [ { api: 'createVolume', @@ -333,7 +339,14 @@ export default { component: shallowRef(defineAsyncComponent(() => import('@/components/view/AnnotationsTab.vue'))) } ], - searchFilters: ['name', 'domainid', 'account', 'tags'], + searchFilters: () => { + var filters = ['name', 'domainid', 'account', 'tags', 'zoneid'] + if (['Admin', 'DomainAdmin'].includes(store.getters.userInfo.roletype)) { + filters.push('storageid') + filters.push('imagestoreid') + } + return filters + }, actions: [ { api: 'createTemplate', diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue index dd7be6bcb27..6c9c13dbf77 100644 --- a/ui/src/views/AutogenView.vue +++ b/ui/src/views/AutogenView.vue @@ -637,7 +637,7 @@ export default { }, watch: { '$route' (to, from) { - if (to.fullPath !== from.fullPath && !to.fullPath.includes('action/')) { + if (to.fullPath !== from.fullPath && !to.fullPath.includes('action/') && to?.query?.tab !== 'browser') { if ('page' in to.query) { this.page = Number(to.query.page) this.pageSize = Number(to.query.pagesize) @@ -788,6 +788,10 @@ export default { this.filters = this.filters() } + if (typeof this.searchFilters === 'function') { + this.searchFilters = this.searchFilters() + } + this.projectView = Boolean(store.getters.project && store.getters.project.id) this.hasProjectId = ['vm', 'vmgroup', 'ssh', 'affinitygroup', 'volume', 'snapshot', 'vmsnapshot', 'guestnetwork', 'vpc', 'securitygroups', 'publicip', 'vpncustomergateway', 'template', 'iso', 'event', 'kubernetes', diff --git a/ui/src/views/image/IsoZones.vue b/ui/src/views/image/IsoZones.vue index 326f1020a6c..daf1e7e21e0 100644 --- a/ui/src/views/image/IsoZones.vue +++ b/ui/src/views/image/IsoZones.vue @@ -34,7 +34,8 @@ :dataSource="dataSource" :pagination="false" :rowSelection="{selectedRowKeys: selectedRowKeys, onChange: onSelectChange}" - :rowKey="record => record.zoneid"> + :rowKey="record => record.zoneid" + :rowExpandable="(record) => record.downloaddetails.length > 0"> + + :rowKey="record => record.zoneid" + :rowExpandable="(record) => record.downloaddetails.length > 0">